Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Gitea Actions
|
cb50f5a6a4 |
@@ -1,31 +0,0 @@
|
||||
# Build artefacts, local configuration, and runtime data never enter an image context.
|
||||
**/node_modules
|
||||
**/.venv
|
||||
**/__pycache__
|
||||
**/.pytest_cache
|
||||
**/dist
|
||||
frontend/prototypes/
|
||||
frontend/vite.database-management-prototype.config.ts
|
||||
**/*.pyc
|
||||
.git
|
||||
.worktrees
|
||||
.thothctl
|
||||
.gitignore
|
||||
**/.env
|
||||
**/.env.*
|
||||
!.env.example
|
||||
!deploy/env/*.env.example
|
||||
deploy/thothii.env
|
||||
deploy/secrets/
|
||||
harness/workspaces/*.yaml
|
||||
!harness/workspaces/local.yaml
|
||||
!harness/workspaces/tht.example.yaml
|
||||
!harness/workspaces/tht-test.yaml
|
||||
**/*.log
|
||||
**/.DS_Store
|
||||
coverage/
|
||||
.coverage
|
||||
.artifacts/
|
||||
data/
|
||||
sessions/
|
||||
workspace-registry/
|
||||
@@ -1,9 +0,0 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
|
||||
[*.ps1]
|
||||
end_of_line = crlf
|
||||
@@ -1,13 +0,0 @@
|
||||
# Common non-secret Compose values. Select local.env or server.env with --env-file.
|
||||
# Run Compose with both files explicitly, for example:
|
||||
# docker compose --env-file deploy/env/local.env -f compose.yaml -f deploy/compose.local.yaml up -d --build
|
||||
|
||||
MAX_PI_PROCESSES=4
|
||||
THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git
|
||||
THT_WORKSPACE_GIT_BRANCH=main
|
||||
THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth Workspace Registry"
|
||||
THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid
|
||||
|
||||
THT_DB_NAME=warehouse
|
||||
THT_DWH_REST_URL=https://dwh.example.invalid
|
||||
THT_LLM_URL=https://llm.example.invalid
|
||||
@@ -1,13 +0,0 @@
|
||||
* text=auto
|
||||
*.sh text eol=lf
|
||||
Dockerfile* text eol=lf
|
||||
*.Dockerfile text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.json text eol=lf
|
||||
*.ts text eol=lf
|
||||
*.tsx text eol=lf
|
||||
*.py text eol=lf
|
||||
*.md text eol=lf
|
||||
*.pptx binary
|
||||
*.ps1 text eol=crlf
|
||||
@@ -1,55 +0,0 @@
|
||||
name: Publish documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "mkdocs.yml"
|
||||
- "docs/requirements.txt"
|
||||
- ".gitea/workflows/publish-docs.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: documentation
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
REPOSITORY_URL: ${{ gitea.server_url }}/${{ gitea.repository }}.git
|
||||
|
||||
steps:
|
||||
- name: Checkout documentation source
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: docs/requirements.txt
|
||||
|
||||
- name: Install MkDocs dependencies
|
||||
run: python -m pip install -r docs/requirements.txt
|
||||
|
||||
- name: Build documentation
|
||||
# Some documented source files intentionally live outside docs/.
|
||||
run: mkdocs build
|
||||
|
||||
- name: Publish generated site to the pages branch
|
||||
working-directory: site
|
||||
run: |
|
||||
git init
|
||||
git config user.name "Gitea Actions"
|
||||
git config user.email "actions@${{ gitea.server_url }}"
|
||||
git add --all
|
||||
git commit --message "Publish documentation for ${{ gitea.sha }}"
|
||||
git -c http.extraheader="Authorization: token ${GITEA_TOKEN}" \
|
||||
push --force "${REPOSITORY_URL}" HEAD:pages
|
||||
@@ -1,34 +0,0 @@
|
||||
name: Container multi-architecture gate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "backend/**"
|
||||
- "frontend/**"
|
||||
- "harness/**"
|
||||
- "docker/**"
|
||||
- "scripts/verify-container-images.sh"
|
||||
- ".github/workflows/container-multiarch.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
driver: docker
|
||||
- name: Build, smoke, security-check, and inventory
|
||||
env:
|
||||
PLATFORM: ${{ matrix.platform }}
|
||||
run: ./scripts/verify-container-images.sh
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: container-inventory-${{ strategy.job-index }}
|
||||
path: .artifacts/container-images/
|
||||
@@ -1,222 +0,0 @@
|
||||
name: Deployment release gate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
windows_docker_startup:
|
||||
description: Run the native self-hosted Windows Docker Desktop/WSL2 release gate
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: deployment-${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deterministic:
|
||||
name: LF, Compose, docs, and TypeScript
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "24.16.0"
|
||||
package-manager-cache: false
|
||||
- name: Install release gate prerequisites
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes --no-install-recommends ripgrep
|
||||
- name: Verify shell syntax and LF policy
|
||||
run: |
|
||||
git ls-files -z '*.sh' | xargs -0 -n1 bash -n
|
||||
bash scripts/verify-line-endings.sh
|
||||
- name: Verify Compose and installation contracts
|
||||
run: |
|
||||
bash scripts/test-unified-compose.sh
|
||||
bash scripts/test-no-deployment-coupling-scope.sh
|
||||
bash scripts/test-compose-secret-policy.sh
|
||||
bash scripts/test-no-deployment-coupling.sh
|
||||
bash scripts/test-verify-workspace-install-docs.sh
|
||||
git diff --check
|
||||
- name: Assert clean checkout before release trust bootstrap
|
||||
run: |
|
||||
git diff --exit-code
|
||||
git diff --cached --exit-code
|
||||
test -z "$(git ls-files --others --exclude-standard)"
|
||||
- name: Verify schema-v3-only release gate
|
||||
run: bash scripts/verify-schema-v3-only-release.sh
|
||||
- name: Verify Task 13 clean-install and runtime fixtures
|
||||
run: |
|
||||
bash scripts/test-server-pi-state-topology.sh
|
||||
bash scripts/unified-deployment-smoke.sh --self-test
|
||||
- name: Install harness CLI for backend integration tests
|
||||
working-directory: harness
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -e .
|
||||
- name: Install backend dependencies
|
||||
working-directory: backend
|
||||
run: npm ci
|
||||
- name: Test and type-check backend
|
||||
working-directory: backend
|
||||
run: |
|
||||
npx vitest run
|
||||
npx tsc --noEmit -p .
|
||||
- name: Install frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
- name: Test and type-check frontend
|
||||
working-directory: frontend
|
||||
run: |
|
||||
npx vitest run
|
||||
npx tsc -b
|
||||
|
||||
authentication-browser:
|
||||
name: Hermetic authentication browser gate
|
||||
needs: deterministic
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "24.16.0"
|
||||
package-manager-cache: false
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/tht/go.sum
|
||||
- name: Install backend dependencies
|
||||
working-directory: backend
|
||||
run: npm ci
|
||||
- name: Install frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
- name: Install Chromium for Playwright
|
||||
working-directory: frontend
|
||||
run: npx playwright install --with-deps chromium
|
||||
- name: Run authentication and authenticated F1 browser smoke
|
||||
run: bash scripts/authentication-smoke.sh
|
||||
|
||||
dwh-auth-linux:
|
||||
name: DWH authentication Nginx gate
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/dwh-auth/go.mod
|
||||
- name: Install Nginx
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes --no-install-recommends nginx-light
|
||||
- name: Run DWH authentication gates
|
||||
run: |
|
||||
(cd tools/dwh-auth && go test -race ./... -count=1 && go vet ./...)
|
||||
bash scripts/test-dwh-auth-build-contract.sh
|
||||
bash scripts/test-dwh-auth-nginx-contract.sh
|
||||
bash scripts/test-dwh-auth-nginx-integration.sh
|
||||
|
||||
linux-docker:
|
||||
name: Linux Docker deployment and rollback
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 100
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install release gate prerequisites
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes --no-install-recommends ripgrep
|
||||
- name: Reclaim unused hosted-runner space
|
||||
run: bash scripts/prepare-linux-docker-runner.sh
|
||||
- name: Run unified deployment smoke
|
||||
env:
|
||||
TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}/task13-images.json
|
||||
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/unified-deployment-smoke.sh
|
||||
- name: Run tht update smoke
|
||||
env:
|
||||
TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}/task13-images.json
|
||||
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/tht-update-smoke.sh
|
||||
- name: Run Linux server deployment smoke
|
||||
env:
|
||||
TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}/task13-images.json
|
||||
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/server-deployment-smoke.sh
|
||||
|
||||
windows-clone:
|
||||
name: Windows clone and Compose contract
|
||||
runs-on: windows-2025
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "24.16.0"
|
||||
package-manager-cache: false
|
||||
- name: Install backend dependencies
|
||||
working-directory: backend
|
||||
run: npm ci
|
||||
- name: Verify clean backend distribution
|
||||
working-directory: backend
|
||||
run: node --test --test-concurrency=1 scripts/clean-dist.test.mjs
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/tht/go.sum
|
||||
- name: Run native Windows retained-capability tests
|
||||
working-directory: tools/tht
|
||||
run: go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1
|
||||
- name: Verify Windows clone contract
|
||||
shell: pwsh
|
||||
run: ./scripts/test-windows-clone-contract.ps1
|
||||
|
||||
windows-docker-release:
|
||||
name: Native Windows Docker Desktop/WSL2 startup
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.windows_docker_startup
|
||||
runs-on: [self-hosted, Windows, X64, docker-desktop]
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/tht/go.sum
|
||||
- name: Run spaced-path Windows Docker release gate
|
||||
shell: pwsh
|
||||
run: ./scripts/test-windows-clone-contract.ps1 -DockerStartup
|
||||
@@ -1,58 +0,0 @@
|
||||
name: Publish documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- gh-pages
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "mkdocs.yml"
|
||||
- ".github/workflows/docs-pages.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
|
||||
steps:
|
||||
- name: Checkout documentation source
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: docs/requirements.txt
|
||||
|
||||
- name: Install MkDocs dependencies
|
||||
run: python -m pip install -r docs/requirements.txt
|
||||
|
||||
- name: Build documentation
|
||||
# The repository intentionally links some docs to source files outside
|
||||
# docs/. MkDocs reports those as warnings, but they must not block Pages.
|
||||
run: mkdocs build
|
||||
|
||||
- name: Configure GitHub Pages
|
||||
uses: actions/configure-pages@v5
|
||||
|
||||
- name: Upload Pages artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: site
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
-87
@@ -1,87 +0,0 @@
|
||||
# === macOS ===
|
||||
.DS_Store
|
||||
|
||||
# === Reference / consultation material (local-only, NOT ThothII deliverables) ===
|
||||
ChironeWp3/
|
||||
Thoth/
|
||||
|
||||
.worktrees/
|
||||
.tht/
|
||||
|
||||
# === Python ===
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
.venv/
|
||||
venv/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# === Node ===
|
||||
node_modules/
|
||||
|
||||
# === Replay bundle (built artifact, like dist/) ===
|
||||
tools/replay/web/
|
||||
|
||||
# === Secrets — NEVER commit ===
|
||||
.env
|
||||
harness/.env
|
||||
harness/workspaces/psd.yaml
|
||||
deploy/thothii.env
|
||||
*.pem
|
||||
ca-chain.pem
|
||||
config/ca-chain.pem
|
||||
|
||||
# ThothII deployment configuration and secret values (keep only the README tracked)
|
||||
deploy/.env
|
||||
deploy/compose.connector-secrets.local.yaml
|
||||
deploy/compose.psd-local.yaml
|
||||
deploy/workspaces/psd.yaml
|
||||
deploy/secrets/*
|
||||
!deploy/secrets/README.md
|
||||
!deploy/secrets/*.example
|
||||
|
||||
# Per-installation configuration generated by `tht setup` (examples stay tracked).
|
||||
deploy/*/thothii-installation.yaml
|
||||
deploy/*/operator.env
|
||||
deploy/*/secrets/*
|
||||
!deploy/*/secrets/.gitkeep
|
||||
!deploy/*/secrets/*.example
|
||||
|
||||
# === Runtime data (sessions contain PII; indexes are derived) ===
|
||||
harness/sessions/
|
||||
harness/indexes/
|
||||
backend/sessions/
|
||||
backend/indexes/
|
||||
|
||||
# === Test artifacts ===
|
||||
.vite/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# === Editor ===
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
|
||||
# Playwright MCP run artifacts
|
||||
.playwright-mcp/
|
||||
|
||||
# === MkDocs build output ===
|
||||
site/
|
||||
|
||||
# Generated container inventory / SBOM-equivalent verification artifacts
|
||||
.artifacts/
|
||||
|
||||
# === PrimeAgent local project settings (per-user, not shared) ===
|
||||
.prime/
|
||||
|
||||
# === Local coding-agent settings (per-user, not shared) ===
|
||||
.commandcode/
|
||||
.reasonix/
|
||||
|
||||
# === Local runtime logs ===
|
||||
logs/
|
||||
@@ -1,203 +0,0 @@
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"generatedAt": "2026-08-26T10:10:15.926Z",
|
||||
"title": "Design System: ThothII",
|
||||
"extensions": {
|
||||
"colorMeta": {
|
||||
"instrument-red": {
|
||||
"role": "primary",
|
||||
"displayName": "Instrument Red",
|
||||
"canonical": "oklch(55.87% 0.1881 23.2)",
|
||||
"tonalRamp": ["oklch(15% 0.07 23.2)", "oklch(28% 0.12 23.2)", "oklch(42% 0.16 23.2)", "oklch(56% 0.1881 23.2)", "oklch(68% 0.17 23.2)", "oklch(78% 0.13 23.2)", "oklch(88% 0.07 23.2)", "oklch(95% 0.03 23.2)"]
|
||||
},
|
||||
"instrument-red-hover": {
|
||||
"role": "primary",
|
||||
"displayName": "Instrument Red Pressed",
|
||||
"canonical": "oklch(50.95% 0.1812 24.1)",
|
||||
"tonalRamp": ["oklch(15% 0.07 24.1)", "oklch(28% 0.12 24.1)", "oklch(42% 0.16 24.1)", "oklch(51% 0.1812 24.1)", "oklch(68% 0.16 24.1)", "oklch(78% 0.12 24.1)", "oklch(88% 0.07 24.1)", "oklch(95% 0.03 24.1)"]
|
||||
},
|
||||
"porcelain-background": {
|
||||
"role": "neutral",
|
||||
"displayName": "Porcelain Background",
|
||||
"canonical": "oklch(99.18% 0.0011 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0011 17.2)", "oklch(28% 0.0011 17.2)", "oklch(42% 0.0011 17.2)", "oklch(56% 0.0011 17.2)", "oklch(68% 0.0011 17.2)", "oklch(78% 0.0011 17.2)", "oklch(88% 0.0011 17.2)", "oklch(95% 0.0011 17.2)"]
|
||||
},
|
||||
"porcelain-card": {
|
||||
"role": "neutral",
|
||||
"displayName": "Porcelain Card",
|
||||
"canonical": "oklch(99.85% 0.0006 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0006 17.2)", "oklch(28% 0.0006 17.2)", "oklch(42% 0.0006 17.2)", "oklch(56% 0.0006 17.2)", "oklch(68% 0.0006 17.2)", "oklch(78% 0.0006 17.2)", "oklch(88% 0.0006 17.2)", "oklch(95% 0.0006 17.2)"]
|
||||
},
|
||||
"warm-surface": {
|
||||
"role": "neutral",
|
||||
"displayName": "Warm Surface",
|
||||
"canonical": "oklch(97.09% 0.0011 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0011 17.2)", "oklch(28% 0.0011 17.2)", "oklch(42% 0.0011 17.2)", "oklch(56% 0.0011 17.2)", "oklch(68% 0.0011 17.2)", "oklch(78% 0.0011 17.2)", "oklch(88% 0.0011 17.2)", "oklch(95% 0.0011 17.2)"]
|
||||
},
|
||||
"sunken-surface": {
|
||||
"role": "neutral",
|
||||
"displayName": "Sunken Surface",
|
||||
"canonical": "oklch(94.08% 0.0011 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0011 17.2)", "oklch(28% 0.0011 17.2)", "oklch(42% 0.0011 17.2)", "oklch(56% 0.0011 17.2)", "oklch(68% 0.0011 17.2)", "oklch(78% 0.0011 17.2)", "oklch(88% 0.0011 17.2)", "oklch(95% 0.0011 17.2)"]
|
||||
},
|
||||
"warm-graphite": {
|
||||
"role": "neutral",
|
||||
"displayName": "Warm Graphite",
|
||||
"canonical": "oklch(26.78% 0.0097 355.6)",
|
||||
"tonalRamp": ["oklch(15% 0.0097 355.6)", "oklch(28% 0.0097 355.6)", "oklch(42% 0.0097 355.6)", "oklch(56% 0.0097 355.6)", "oklch(68% 0.008 355.6)", "oklch(78% 0.006 355.6)", "oklch(88% 0.004 355.6)", "oklch(95% 0.002 355.6)"]
|
||||
},
|
||||
"muted-graphite": {
|
||||
"role": "neutral",
|
||||
"displayName": "Muted Graphite",
|
||||
"canonical": "oklch(51.33% 0.0088 345.6)",
|
||||
"tonalRamp": ["oklch(15% 0.0088 345.6)", "oklch(28% 0.0088 345.6)", "oklch(42% 0.0088 345.6)", "oklch(56% 0.0088 345.6)", "oklch(68% 0.007 345.6)", "oklch(78% 0.005 345.6)", "oklch(88% 0.003 345.6)", "oklch(95% 0.002 345.6)"]
|
||||
},
|
||||
"quiet-border": {
|
||||
"role": "neutral",
|
||||
"displayName": "Quiet Border",
|
||||
"canonical": "oklch(90.93% 0.0035 354.7)",
|
||||
"tonalRamp": ["oklch(15% 0.0035 354.7)", "oklch(28% 0.0035 354.7)", "oklch(42% 0.0035 354.7)", "oklch(56% 0.0035 354.7)", "oklch(68% 0.0035 354.7)", "oklch(78% 0.0035 354.7)", "oklch(88% 0.003 354.7)", "oklch(95% 0.002 354.7)"]
|
||||
},
|
||||
"success-mint": {
|
||||
"role": "secondary",
|
||||
"displayName": "Success Mint",
|
||||
"canonical": "oklch(75.77% 0.1581 165)",
|
||||
"tonalRamp": ["oklch(15% 0.06 165)", "oklch(28% 0.1 165)", "oklch(42% 0.14 165)", "oklch(56% 0.1581 165)", "oklch(68% 0.15 165)", "oklch(78% 0.12 165)", "oklch(88% 0.07 165)", "oklch(95% 0.03 165)"]
|
||||
},
|
||||
"warning-amber": {
|
||||
"role": "tertiary",
|
||||
"displayName": "Warning Amber",
|
||||
"canonical": "oklch(85.23% 0.1386 78.9)",
|
||||
"tonalRamp": ["oklch(15% 0.05 78.9)", "oklch(28% 0.09 78.9)", "oklch(42% 0.12 78.9)", "oklch(56% 0.1386 78.9)", "oklch(68% 0.13 78.9)", "oklch(78% 0.1 78.9)", "oklch(88% 0.06 78.9)", "oklch(95% 0.025 78.9)"]
|
||||
},
|
||||
"information-blue": {
|
||||
"role": "tertiary",
|
||||
"displayName": "Information Blue",
|
||||
"canonical": "oklch(70.35% 0.1128 221.3)",
|
||||
"tonalRamp": ["oklch(15% 0.045 221.3)", "oklch(28% 0.075 221.3)", "oklch(42% 0.1 221.3)", "oklch(56% 0.1128 221.3)", "oklch(68% 0.105 221.3)", "oklch(78% 0.08 221.3)", "oklch(88% 0.045 221.3)", "oklch(95% 0.02 221.3)"]
|
||||
}
|
||||
},
|
||||
"typographyMeta": {
|
||||
"display": {"displayName": "Display", "purpose": "Authentication and exceptional page-level statements only."},
|
||||
"headline": {"displayName": "Headline", "purpose": "Major page and persisted artifact titles."},
|
||||
"title": {"displayName": "Title", "purpose": "Panel and document section hierarchy."},
|
||||
"body": {"displayName": "Body", "purpose": "Operational prose and sustained reading."},
|
||||
"control": {"displayName": "Control", "purpose": "Buttons, inputs, tabs, and compact actions."},
|
||||
"label": {"displayName": "Machine Label", "purpose": "Uppercase metadata and machine-oriented micro-labels."}
|
||||
},
|
||||
"shadows": [
|
||||
{"name": "contact", "value": "0 1px 2px oklch(var(--shadow-tint) / 0.05)", "purpose": "Contact shadow for controls and code blocks."},
|
||||
{"name": "panel", "value": "0 1px 2px oklch(var(--shadow-tint) / 0.05), 0 2px 6px -1px oklch(var(--shadow-tint) / 0.05)", "purpose": "Small structural lift for selected cards."},
|
||||
{"name": "overlay", "value": "0 2px 4px -2px oklch(var(--shadow-tint) / 0.06), 0 12px 32px -8px oklch(var(--shadow-tint) / 0.1)", "purpose": "Broad low-opacity lift for dialogs and floating layers."}
|
||||
],
|
||||
"motion": [
|
||||
{"name": "control-feedback", "value": "140ms cubic-bezier(0.22, 1, 0.36, 1)", "purpose": "Button hover, focus, and press feedback."},
|
||||
{"name": "overlay-transition", "value": "100ms ease-out", "purpose": "Dialog fade and scale transitions."},
|
||||
{"name": "activity-pulse", "value": "1.5s ease-in-out infinite", "purpose": "Live model activity only; disabled for reduced motion."}
|
||||
],
|
||||
"breakpoints": [
|
||||
{"name": "sm", "value": "640px"},
|
||||
{"name": "lg", "value": "1024px"}
|
||||
]
|
||||
},
|
||||
"components": [
|
||||
{
|
||||
"name": "Primary Button",
|
||||
"kind": "button",
|
||||
"refersTo": "button-primary",
|
||||
"description": "The authoritative action for the current workflow step.",
|
||||
"html": "<button class=\"ds-button-primary\">Confirm review</button>",
|
||||
"css": ".ds-button-primary { display:inline-flex; align-items:center; justify-content:center; height:32px; padding:0 14px; border:1px solid transparent; border-radius:8px; background:oklch(var(--primary)); color:oklch(var(--primary-foreground)); font:600 14px/1.25 var(--font-sans); letter-spacing:0.005em; box-shadow:var(--shadow-xs); transition:color 140ms cubic-bezier(0.22,1,0.36,1),background-color 140ms cubic-bezier(0.22,1,0.36,1),box-shadow 140ms cubic-bezier(0.22,1,0.36,1),transform 140ms cubic-bezier(0.22,1,0.36,1); } .ds-button-primary:hover { background:oklch(var(--primary-hover)); } .ds-button-primary:focus-visible { outline:3px solid oklch(var(--ring)/0.25); outline-offset:2px; } .ds-button-primary:active { transform:scale(0.97); box-shadow:none; }"
|
||||
},
|
||||
{
|
||||
"name": "Outline Button",
|
||||
"kind": "button",
|
||||
"refersTo": "button-secondary",
|
||||
"description": "A compact secondary action that preserves the primary action hierarchy.",
|
||||
"html": "<button class=\"ds-button-outline\">Inspect details</button>",
|
||||
"css": ".ds-button-outline { display:inline-flex; align-items:center; justify-content:center; height:32px; padding:0 14px; border:1px solid oklch(var(--border)); border-radius:8px; background:oklch(var(--card)); color:oklch(var(--foreground)); font:600 14px/1.25 var(--font-sans); box-shadow:var(--shadow-xs); transition:background-color 140ms cubic-bezier(0.22,1,0.36,1),transform 140ms cubic-bezier(0.22,1,0.36,1); } .ds-button-outline:hover { background:oklch(var(--muted)); } .ds-button-outline:focus-visible { outline:3px solid oklch(var(--ring)/0.25); outline-offset:2px; } .ds-button-outline:active { transform:scale(0.97); box-shadow:none; }"
|
||||
},
|
||||
{
|
||||
"name": "Status Badge",
|
||||
"kind": "chip",
|
||||
"refersTo": "badge-primary",
|
||||
"description": "A compact state label that always carries readable text.",
|
||||
"html": "<span class=\"ds-status-badge\">Ready for review</span>",
|
||||
"css": ".ds-status-badge { display:inline-flex; align-items:center; height:20px; padding:2px 8px; border:1px solid transparent; border-radius:6px; background:oklch(var(--primary)); color:oklch(var(--primary-foreground)); font:600 12px/1.25 var(--font-sans); white-space:nowrap; } .ds-status-badge:focus-visible { outline:3px solid oklch(var(--ring)/0.5); outline-offset:2px; }"
|
||||
},
|
||||
{
|
||||
"name": "Text Field",
|
||||
"kind": "input",
|
||||
"refersTo": "input-default",
|
||||
"description": "A readable operational field with an explicit focus state.",
|
||||
"html": "<input class=\"ds-text-field\" value=\"Fascia pediatrica\" aria-label=\"Session name\">",
|
||||
"css": ".ds-text-field { width:280px; height:40px; padding:0 12px; border:1px solid oklch(var(--input)); border-radius:8px; background:oklch(var(--background)); color:oklch(var(--foreground)); font:400 14px/1.5 var(--font-sans); outline:none; } .ds-text-field:hover { border-color:oklch(var(--muted-foreground)/0.65); } .ds-text-field:focus-visible { border-color:oklch(var(--ring)); box-shadow:0 0 0 3px oklch(var(--ring)/0.25); } .ds-text-field:disabled { opacity:0.5; cursor:not-allowed; }"
|
||||
},
|
||||
{
|
||||
"name": "Work Card",
|
||||
"kind": "card",
|
||||
"refersTo": "card-default",
|
||||
"description": "A single-level container for a coherent review surface.",
|
||||
"html": "<section class=\"ds-work-card\"><h3>Schema linking</h3><p>Review the linked tables and columns before continuing.</p></section>",
|
||||
"css": ".ds-work-card { width:320px; padding:16px; border:1px solid oklch(var(--border)/0.7); border-radius:12px; background:oklch(var(--card)); color:oklch(var(--card-foreground)); box-shadow:var(--shadow-sm); } .ds-work-card h3 { margin:0 0 8px; font:500 16px/1.35 var(--font-heading); letter-spacing:-0.01em; } .ds-work-card p { margin:0; color:oklch(var(--muted-foreground)); font:400 14px/1.6 var(--font-sans); } .ds-work-card:focus-within { outline:3px solid oklch(var(--ring)/0.25); outline-offset:2px; }"
|
||||
},
|
||||
{
|
||||
"name": "Session Navigation Item",
|
||||
"kind": "nav",
|
||||
"description": "A dense session row with restrained hover and active hierarchy.",
|
||||
"html": "<button class=\"ds-session-item\"><span class=\"ds-session-dot\"></span><span><strong>Patient cohorts</strong><small>Schema linking</small></span></button>",
|
||||
"css": ".ds-session-item { display:flex; width:260px; align-items:center; gap:8px; padding:4px 8px; border:0; border-radius:8px; background:transparent; color:oklch(var(--foreground)); text-align:left; font-family:var(--font-sans); transition:background-color 140ms cubic-bezier(0.22,1,0.36,1); } .ds-session-item:hover,.ds-session-item[aria-current=\"page\"] { background:oklch(var(--accent)); } .ds-session-item:focus-visible { outline:2px solid oklch(var(--ring)/0.4); outline-offset:1px; } .ds-session-dot { width:6px; height:6px; flex:none; border-radius:9999px; background:oklch(var(--success)); } .ds-session-item strong,.ds-session-item small { display:block; } .ds-session-item strong { font-size:13px; font-weight:600; } .ds-session-item small { margin-top:2px; color:oklch(var(--muted-foreground)); font-size:11px; }"
|
||||
},
|
||||
{
|
||||
"name": "Curated Evidence Document",
|
||||
"kind": "custom",
|
||||
"description": "The table-free reading hierarchy for persisted evidence.",
|
||||
"html": "<article class=\"ds-evidence\"><h2>Fascia pediatrica</h2><div class=\"ds-evidence-summary\"><strong>Dominio</strong> · Italiano<br><span>Scopi: Disambiguazione · Generazione SQL</span></div><h3>Ambito di applicazione</h3><ul><li>fascia pediatrica</li><li>paziente minore</li></ul><h3>Regola</h3><p>La fascia pediatrica comprende i pazienti con età inferiore a 18 anni.</p><details><summary>Dettagli tecnici e provenienza</summary><code>evidence:fascia-pediatrica</code></details></article>",
|
||||
"css": ".ds-evidence { max-width:70ch; color:oklch(var(--foreground)); font:400 15px/1.65 var(--font-sans); } .ds-evidence h2,.ds-evidence h3 { font-family:var(--font-heading); letter-spacing:-0.01em; } .ds-evidence h2 { margin:0 0 16px; font-size:24px; } .ds-evidence h3 { margin:24px 0 8px; font-size:18px; } .ds-evidence-summary { padding:12px 14px; border:1px solid oklch(var(--border)); border-radius:8px; background:oklch(var(--muted)); color:oklch(var(--muted-foreground)); } .ds-evidence-summary strong { color:oklch(var(--foreground)); } .ds-evidence ul { padding-left:20px; } .ds-evidence details { margin-top:24px; padding:10px 12px; border:1px solid oklch(var(--border)); border-radius:8px; background:oklch(var(--card)); } .ds-evidence summary { cursor:pointer; font-weight:600; } .ds-evidence code { font-family:var(--font-mono); }"
|
||||
}
|
||||
],
|
||||
"narrative": {
|
||||
"northStar": "The Clinical Workbench",
|
||||
"overview": "ThothII should feel like a well-kept clinical workbench: warm enough for sustained reading, exact enough for consequential review, and quiet enough that evidence, state, and decisions remain in the foreground. The visual system is calm, precise, and trustworthy. It uses familiar product patterns, restrained color, and deliberate density instead of decorative spectacle.\n\nThe primary physical scene is an analyst reviewing persisted evidence and SQL on a large monitor in a well-lit working environment. This makes the warm light theme the default. The supported dark theme serves lower-light work without becoming a separate neon aesthetic. Both themes preserve the same hierarchy and semantic roles.\n\nThe system rejects generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long choreographed transitions, and effects that compete with the analytical task. Controls should feel disciplined and tactile, never playful, sluggish, or visually unstable.",
|
||||
"keyCharacteristics": [
|
||||
"Warm, restrained surfaces with one scarce red accent.",
|
||||
"Editorial headings paired with highly legible operational body text.",
|
||||
"Dense information organized through hierarchy, rhythm, and progressive disclosure.",
|
||||
"Persisted artifacts and reviewer decisions presented as the visual source of truth.",
|
||||
"Fast state feedback with reduced-motion parity."
|
||||
],
|
||||
"rules": [
|
||||
{"name": "The Workbench Rule", "body": "Every visual element must support inspection, action, state, or provenance. Decoration without an operational purpose is forbidden.", "section": "overview"},
|
||||
{"name": "The Persisted Truth Rule", "body": "Persisted artifacts and reviewer decisions receive stronger hierarchy than transient model narration.", "section": "overview"},
|
||||
{"name": "The Density with Rhythm Rule", "body": "Preserve information density, but vary spacing between groups so users can scan structure without adding nested containers.", "section": "overview"},
|
||||
{"name": "The One Voice Rule", "body": "Instrument Red should occupy no more than roughly ten percent of a screen. Its rarity is what makes it authoritative.", "section": "colors"},
|
||||
{"name": "The State Has a Name Rule", "body": "Success, warning, information, and destructive colors are reserved for their named states. Color is never the only state indicator.", "section": "colors"},
|
||||
{"name": "The Three Registers Rule", "body": "Serif means authority, sans means interaction and reading, mono means machine identity. Do not exchange these roles for novelty.", "section": "typography"},
|
||||
{"name": "The Read Once Rule", "body": "A heading, label, and body must be distinguishable on first glance through size and weight. Do not repeat headings in explanatory copy.", "section": "typography"},
|
||||
{"name": "The Flat by Default Rule", "body": "A resting surface has no shadow unless it is physically above another surface. If every panel floats, none of them has hierarchy.", "section": "elevation"},
|
||||
{"name": "The Borders Structure, Shadows Elevate Rule", "body": "Never use shadow as a substitute for grouping or a border as a decorative accent.", "section": "elevation"},
|
||||
{"name": "The Review Surface Rule", "body": "The visible Markdown must be readable without understanding the machine contract. Technical metadata belongs in progressive disclosure, not above the title.", "section": "components"}
|
||||
],
|
||||
"dos": [
|
||||
"Do make every state change unmistakable without interrupting flow.",
|
||||
"Do use Instrument Red only for primary action, current selection, focus identity, or explicit destructive meaning.",
|
||||
"Do preserve information density with headings, rhythm, and progressive disclosure.",
|
||||
"Do keep keyboard focus explicit and pair color with text, shape, icon, or position.",
|
||||
"Do respect prefers-reduced-motion while preserving immediate non-kinetic feedback.",
|
||||
"Do use English for interface chrome and the workspace language for persisted document content.",
|
||||
"Do render curated metadata and scope as Markdown prose or lists, never as a frontmatter table."
|
||||
],
|
||||
"donts": [
|
||||
"Don't add generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long choreographed transitions, or effects that compete with the analytical task.",
|
||||
"Don't make controls feel playful, sluggish, or visually unstable.",
|
||||
"Don't use gradient text, decorative glassmorphism, or full-saturation accents on inactive states.",
|
||||
"Don't use a colored side stripe greater than one pixel on cards, callouts, list items, or blockquotes. Use a full border, tonal background, icon, or heading instead.",
|
||||
"Don't nest cards or wrap every section in a container.",
|
||||
"Don't use a modal before exhausting inline or progressive alternatives.",
|
||||
"Don't use tables for applies_to, metadata, enum values, or other one-dimensional content.",
|
||||
"Don't use color as the sole carrier of success, warning, error, selection, or progress.",
|
||||
"Don't use display typography for buttons, labels, or data.",
|
||||
"Don't add em dashes to interface copy. Use commas, colons, semicolons, or parentheses."
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
|
||||
|
||||
<link rel="shortcut icon" href="/thothii-docs//img/favicon.ico">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0" />
|
||||
<title>ThothII Docs</title>
|
||||
<link href="/thothii-docs//css/bootstrap-3.3.7.min.css" rel="stylesheet">
|
||||
<link href="/thothii-docs//css/font-awesome-4.7.0.css" rel="stylesheet">
|
||||
<link href="/thothii-docs//css/base.css" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/thothii-docs//css/highlight.css">
|
||||
<link href="/thothii-docs/stylesheets/extra.css" rel="stylesheet">
|
||||
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
|
||||
<!--[if lt IE 9]>
|
||||
<script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
|
||||
<script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script src="/thothii-docs//js/jquery-3.2.1.min.js"></script>
|
||||
<script src="/thothii-docs//js/bootstrap-3.3.7.min.js"></script>
|
||||
<script src="/thothii-docs//js/highlight.pack.js"></script>
|
||||
|
||||
<base target="_top">
|
||||
<script>
|
||||
var base_url = '/thothii-docs/';
|
||||
var is_top_frame = false;
|
||||
var page_toc = null;
|
||||
|
||||
</script>
|
||||
<script src="/thothii-docs//js/base.js"></script>
|
||||
<script src="/thothii-docs/javascripts/layout-init.js"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<script>
|
||||
if (is_top_frame) { $('body').addClass('wm-top-page'); }
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
<div class="container-fluid wm-page-content">
|
||||
<a name="_top"></a>
|
||||
|
||||
|
||||
|
||||
<h2 style="text-align: center">404</h2>
|
||||
<h1 style="text-align: center">Page not found</h1>
|
||||
|
||||
|
||||
|
||||
<br>
|
||||
|
||||
|
||||
<br>
|
||||
</div>
|
||||
|
||||
<footer class="container-fluid wm-page-content">
|
||||
<p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a> using <a href="https://github.com/gristlabs/mkdocs-windmill">Windmill</a> theme by Grist Labs.</p>
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,110 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
## Agent skills
|
||||
|
||||
### Issue tracker
|
||||
|
||||
Issues for this repository live in the self-hosted Gitea repository at `https://git.tylconsulting.it/mptyl/ThothII`; use its web UI or authenticated Gitea API. See `docs/agents/issue-tracker.md`.
|
||||
|
||||
### Triage labels
|
||||
|
||||
Use the canonical labels `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, and `wontfix`. See `docs/agents/triage-labels.md`.
|
||||
|
||||
### Domain docs
|
||||
|
||||
This is a single-context repository with root `CONTEXT.md` and `docs/adr/`. See `docs/agents/domain.md`.
|
||||
|
||||
This file provides guidance to Codex (Codex.ai/code) when working with code in this repository.
|
||||
|
||||
## Start here
|
||||
|
||||
Read [PROJECT_STATE.md](PROJECT_STATE.md) for the current-state snapshot: what was last
|
||||
built, pending manual gates, workspace/secret layout, and design-doc locations. This file
|
||||
holds the stable commands + architecture mental model; PROJECT_STATE.md holds the evolving
|
||||
detail. Current architecture and contracts live in `docs/architecture/`, `docs/contracts/`,
|
||||
and `docs/evidence.md`; durable design decisions live in `docs/adr/`. Git history is the source
|
||||
for superseded designs and implementation plans.
|
||||
|
||||
## Commands
|
||||
|
||||
The repo has three independently-built layers. Run the local Docker stack with `./scripts/run-stack.sh` after creating `deploy/env/local.env`; it starts the base+local Compose profile with `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. The core image contains Pi. Qdrant and Ollama are internal Compose services; DWH and LLM remain external configuration endpoints.
|
||||
|
||||
**Native host CLI `tht`** (`tools/tht/`)
|
||||
- Operator surface: `setup`, `start`, `stop`, `status`, `doctor`, `auth`, `workspace`, and `pi`.
|
||||
- Use `tht --installation <absolute-path>/thothii-installation.yaml <command>` for installation,
|
||||
authentication, diagnostics, lifecycle, and workspace operations.
|
||||
|
||||
**harness/** (Python workflow `tht` CLI + Pi gate extension)
|
||||
- Install: `cd harness && python -m venv .venv && pip install -e ".[dev]"` (puts `tht` on PATH)
|
||||
- Test: `.venv/bin/pytest -q` — `l2` (real GLM + remote DB) is opt-in via `addopts = -m 'not l2'`; `l0` (testcontainers) needs Docker
|
||||
- Single test: `.venv/bin/pytest tests/test_session_mutations.py::test_set_name -v` (or `-k <pattern>`); include e2e with `-m l2`
|
||||
- Lint: `.venv/bin/ruff check .` (line-length 100)
|
||||
|
||||
**backend/** (Fastify + TypeScript, vitest)
|
||||
- Dev: `npm run dev` (tsx watch `src/server.ts`) · Build: `npm run build` (tsc → `dist/`)
|
||||
- Test: `npx vitest run` · Single: `npx vitest run test/routes-sessions.test.ts -t "rename"`
|
||||
- Typecheck: `npx tsc --noEmit -p .` (vitest does NOT type-check — run this before committing)
|
||||
|
||||
**frontend/** (React 18 + Vite + vitest)
|
||||
- Dev: `npm run dev` (Vite; set `VITE_BACKEND_URL`) · Build: `npm run build`
|
||||
- Test: `npx vitest run` · Single: `npx vitest run src/shell/NavSessions.test.tsx`
|
||||
- Typecheck: `npx tsc -b` · E2E: `npm run e2e` (Playwright)
|
||||
|
||||
**Documentation** (MkDocs, repository-locked Python dependencies)
|
||||
- Strict build: `./scripts/build-docs.sh`
|
||||
- Refresh lock: `./scripts/update-docs-lock.sh`
|
||||
|
||||
No ESLint on the TS layers — `tsc` is the gate. Tests use vitest + MSW (no network).
|
||||
|
||||
## Architecture (the parts that need multiple files to see)
|
||||
|
||||
```
|
||||
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
|
||||
```
|
||||
|
||||
- **The harness owns the workflow and all persistence.** The Python workflow CLI `tht` inside
|
||||
`core` is deterministic; `harness/.pi/extensions/tht-gate.js` is a Pi extension that drives an **8-phase
|
||||
NL→SQL workflow**. The single source of workflow truth is `harness/workflow.yaml`; the
|
||||
orchestration rules the model must follow are `harness/.pi/skills/tht-sessione/SKILL.md`.
|
||||
"Current phase" is computed by folding the decision ledger (`harness/tht/phase.py`), not
|
||||
stored — read it before reasoning about phase logic.
|
||||
|
||||
- **Persistence = phase documents, NOT chat.** A session is a directory under the workspace's
|
||||
`sessions/` path: `session_manifest.yaml` + per-phase artifacts (`question.md`,
|
||||
`schema_linking.json`, `sql_final.sql`, …) + `review_decisions.jsonl`. The contract
|
||||
(SKILL.md): *"the persisted state is the truth — what is not recorded did not happen."*
|
||||
There is no verbatim transcript store. A resumed Pi process rebuilds context from
|
||||
`tht session show <id>` + the on-disk artifacts.
|
||||
|
||||
- **The backend bridges sessions and owns the installation-local metadata catalog.** `ThtRunner`
|
||||
shells the Python workflow `tht` subcommands inside `core`;
|
||||
`PiProcessManager` runs one Pi child per session and bridges its RPC stream;
|
||||
`SessionBridge` maps Pi RPC events → client events (`ui_request`/`text_delta`/`info`);
|
||||
`SseHub` fans them out over SSE to the browser. The separate PostgreSQL catalog stores database
|
||||
metadata and sequential AI description-generation runs. Description generation samples the DWH
|
||||
through read-only connectors and calls a short-lived Python LiteLLM helper; it does not use Pi or
|
||||
expose a public CLI command. App settings still live in `backend/data/settings.json`.
|
||||
|
||||
- **Human-in-the-loop gate contract.** The model proposes; a human reviewer decides at gates
|
||||
via widgets (`reviewer_select` = single pick — a chosen option carrying a `decision` payload
|
||||
auto-confirms/persists directly, an option without one only asks; `reviewer_decide` = multiselect,
|
||||
each choice IS a decision; `reviewer_confirm` = artifact/phase gate). The frontend renders these
|
||||
widget-descriptors (`src/widgets/` registry) and the live transcript is rebuilt in-memory
|
||||
from the SSE stream (`src/store/sessionStore.ts`) — it is not persisted.
|
||||
|
||||
## Project-specific gotchas
|
||||
|
||||
- **`tht`'s `-c`/`--config` is a PER-COMMAND option** — it must follow the subcommand, never
|
||||
precede it (`ThtRunner.buildArgv` enforces this; prepending caused live 500s).
|
||||
- **`--json` output must be pristine** (only valid JSON on stdout) — used as a machine contract.
|
||||
- **UI strings are English; document *content* stays the workspace language** (Italian for
|
||||
`psd`) because it's the real data. Only chrome/labels are English.
|
||||
- **Workspaces** (`harness/workspaces/*.yaml`) set the DB target and **absolute**
|
||||
`paths.sessions/artifacts/indexes` — for `psd` these point at a *separate, uncommitted* repo
|
||||
(`tht-workspace-psd/`). Secrets live ONLY in `harness/.env` (gitignored).
|
||||
- **Settings are global** (`backend/data/settings.json`: workspace/provider/model/thinking);
|
||||
the New-session form is question-only.
|
||||
- **Resume**: a resumable session re-enters at its last incomplete phase. The backend refuses
|
||||
resume with 409 when `finalized` or `archived`, and `PiProcessManager.spawnFor` must send
|
||||
`/riprendi-sessione <id>` (resume mode) vs `/nuova-domanda` (new) — sending the wrong prompt
|
||||
silently turns a resume into a new question.
|
||||
-464
@@ -1,464 +0,0 @@
|
||||
# Contesto di dominio di ThothII
|
||||
|
||||
## Architettura del workflow
|
||||
|
||||
**Workflow Kernel** — Il coordinatore deterministico che possiede lo stato del workflow,
|
||||
le transizioni, il rollback, la finalizzazione e l'applicazione atomica degli esiti dei
|
||||
moduli.
|
||||
|
||||
**Workflow Module** — Una capacità incapsulata che espone un contratto versionato. Un
|
||||
modulo può partecipare a più stage e non modifica direttamente lo stato del workflow.
|
||||
|
||||
**Stage** — Un punto del workflow, identificato semanticamente, nel quale viene invocato
|
||||
un modulo. L'identità dello stage è indipendente dalla sua posizione visiva.
|
||||
|
||||
**Display code** — L'etichetta di presentazione associata a uno stage, per esempio da
|
||||
`F1` a `F8`. I display code alimentano gli indicatori di avanzamento nel frontend, ma non
|
||||
sono usati come identità del workflow o chiavi di dipendenza.
|
||||
|
||||
**Module outcome** — Il risultato proposto da un modulo: eventi tipizzati, modifiche agli
|
||||
artifact, un'eventuale richiesta di revisione umana e uno stato di esecuzione. Il
|
||||
Workflow Kernel valida e applica l'esito.
|
||||
|
||||
**Revision request** — La proposta tipizzata con cui un modulo segnala che lo stage
|
||||
corrente non può concludersi validamente senza rieseguire lo stesso stage o uno stage
|
||||
precedente. Non produce direttamente una transizione: il Workflow Kernel valida la
|
||||
richiesta, sospende l'avanzamento e, per riaprire uno stage già completato, attende una
|
||||
decisione umana tipizzata. Il Kernel, non il modulo, determina gli eventi e gli artifact
|
||||
causalmente da rendere stale.
|
||||
|
||||
**Question Admission** — Il controllo preliminare eseguito prima delle fasi da `F1` a
|
||||
`F8`. Nella prima release distingue una domanda utilizzabile da input garbage e verifica
|
||||
che la domanda appartenga allo scope dichiarato dal workspace. Il suo stato è mostrato
|
||||
separatamente dagli otto indicatori di fase.
|
||||
|
||||
**Workspace scope** — La dichiarazione gestita e versionata di ciò che il database di un
|
||||
workspace rappresenta e delle domande alle quali è destinato a rispondere. Question
|
||||
Admission la usa come riferimento per valutare la pertinenza di una domanda.
|
||||
|
||||
**Datamart Plugin** — Il modulo sostituibile che implementa lo stage semantico
|
||||
`datamart`, presentato con display code `F8`. La promozione della memory e la
|
||||
finalizzazione della sessione non appartengono al Datamart Plugin.
|
||||
|
||||
**Ordered workflow** — La pipeline deterministica composta dal preflight Admission,
|
||||
dagli otto stage principali ordinati da `F1` a `F8` e dalla finalizzazione. L'ordine
|
||||
degli stage è esplicito; il workflow non è un DAG generale.
|
||||
|
||||
**Extension point** — Una posizione semantica nel lifecycle dell'Ordered workflow alla
|
||||
quale possono contribuire uno o più moduli senza diventare nuovi stage visibili. Un
|
||||
extension point non possiede un display code.
|
||||
|
||||
**Stage state** — La proiezione deterministica degli eventi del workflow che descrive
|
||||
uno stage come `pending`, `ready`, `running`, `awaiting_human`, `completed`, `skipped` o
|
||||
`failed`. Non è un valore corrente memorizzato separatamente dal ledger.
|
||||
|
||||
**Required contribution** — Il contributo di un modulo a un extension point che deve
|
||||
concludersi o essere esplicitamente saltato secondo policy prima che il workflow possa
|
||||
avanzare.
|
||||
|
||||
**Best-effort contribution** — Il contributo di un modulo il cui fallimento viene
|
||||
registrato e mostrato come warning, ma non impedisce al workflow di avanzare.
|
||||
|
||||
**Blocked workflow** — La proiezione complessiva di un workflow che non può avanzare a
|
||||
causa di uno stage o di un contributo required fallito o non disponibile. `Blocked` non
|
||||
è uno Stage state autonomo.
|
||||
|
||||
**Module invocation** — Una singola richiesta del Workflow Kernel a un modulo in uno
|
||||
stage o extension point. Conserva la stessa identità attraverso eventuali retry, che
|
||||
sono tentativi distinti della medesima invocation.
|
||||
|
||||
**Stage skip** — La conclusione esplicita di uno stage senza eseguirne il comportamento.
|
||||
È ammessa soltanto dalla policy dello stage e registra motivo e attore; un fallimento non
|
||||
equivale mai implicitamente a uno skip.
|
||||
|
||||
**Stage reopen** — La riapertura di uno stage non finalizzato che rende stale gli esiti
|
||||
causalmente successivi. Gli effetti esterni già prodotti richiedono una marcatura o una
|
||||
compensazione esplicita e non sono presentati come automaticamente annullati. Può essere
|
||||
applicata dal Workflow Kernel in seguito all'approvazione di una Revision request, ma
|
||||
non può essere eseguita direttamente da Pi o da un Workflow Module.
|
||||
|
||||
**Completion policy** — La regola con cui uno stage si conclude: `automatic` quando il
|
||||
kernel può verificarne deterministicamente l'esito, oppure `review_required` quando è
|
||||
necessaria un'approvazione umana tipizzata.
|
||||
|
||||
**Paused session** — Una sessione interrotta intenzionalmente ma resumibile. L'azione
|
||||
“Stop and save” mette la sessione in pausa; non la completa e non la marca come fallita.
|
||||
|
||||
**Finalized session** — Una sessione completata con esito canonico e immutabile. Una
|
||||
correzione successiva crea una nuova sessione derivata, collegata a quella precedente.
|
||||
|
||||
**After-finalize hook** — Una notifica o attività best-effort eseguita tramite outbox
|
||||
dopo la finalizzazione. Non può modificare il ledger, gli artifact canonici o lo stato
|
||||
terminale della sessione.
|
||||
|
||||
## Evidence
|
||||
|
||||
**Evidence Module** — Il modulo autonomo che possiede la preparazione delle Evidence e
|
||||
la loro consultazione durante il workflow. La preparazione avviene fuori dalle singole
|
||||
sessioni; il workflow usa soltanto contenuti già pubblicati. A runtime contribuisce agli
|
||||
stage semantici esistenti, senza diventare uno stage visibile e senza modificare ledger,
|
||||
artifact o stato del workflow.
|
||||
|
||||
**Source Evidence** — Un documento originale del workspace, conservato senza modifiche
|
||||
come riferimento umano e origine della successiva ristrutturazione.
|
||||
|
||||
**Evidence Unit** — La più piccola unità semantica coerente, revisionabile e ricercabile
|
||||
derivata da una sola Source Evidence. Possiede un identificatore stabile indipendente
|
||||
dal kind, assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono
|
||||
fuse automaticamente.
|
||||
|
||||
**Evidence kind** — La categoria semantica di una Evidence Unit, che ne determina i
|
||||
campi specifici e ne orienta l'uso. Ogni unità ha un solo kind primario; i tipi iniziali
|
||||
sono `glossary`, `domain`, `enum`, `example`, `mapping`, `normalization`, `formula` e
|
||||
`reference`.
|
||||
|
||||
**Glossary Evidence** — Una Evidence Unit che definisce il significato linguistico, i
|
||||
sinonimi o le varianti di un termine.
|
||||
|
||||
**Domain Evidence** — Una Evidence Unit che esprime una regola o un vincolo del dominio
|
||||
non rappresentato da un kind più specifico.
|
||||
|
||||
**Enum Evidence** — Una Evidence Unit che collega un insieme finito di valori
|
||||
memorizzati ai relativi significati.
|
||||
|
||||
**Example Evidence** — Una Evidence Unit che associa un input o una domanda alla sua
|
||||
interpretazione o al risultato atteso.
|
||||
|
||||
**Mapping Evidence** — Una Evidence Unit che collega un concetto logico agli elementi
|
||||
del relativo schema fisico.
|
||||
|
||||
**Normalization Evidence** — Una Evidence Unit che descrive la trasformazione di una
|
||||
rappresentazione in una forma canonica.
|
||||
|
||||
**Formula Evidence** — Una Evidence Unit che contiene una singola espressione PostgreSQL
|
||||
componibile e ne dichiara gli input. Una query SQL completa non è una Formula Evidence.
|
||||
|
||||
**Reference Evidence** — Una Evidence Unit che rappresenta un collegamento esterno da
|
||||
restituire come contenuto autonomo, anziché come semplice provenienza.
|
||||
|
||||
**Evidence purpose** — La destinazione dichiarata di una Evidence Unit nel workflow:
|
||||
disambiguation, rewriting, schema linking o SQL generation. È distinta dall'Evidence
|
||||
kind: il tipo descrive cosa contiene, il purpose quando può essere utile; durante la
|
||||
ricerca il purpose richiesto è un filtro obbligatorio. Il recupero di esperienze e
|
||||
soluzioni precedenti appartiene al Memory Module e non è un Evidence purpose.
|
||||
|
||||
**Evidence Search Outcome** — Il risultato tipizzato di una consultazione del modulo
|
||||
Evidence. Distingue una ricerca disponibile, che può legittimamente non trovare
|
||||
corrispondenze, da un'indisponibilità tecnica che impedisce allo stage chiamante di
|
||||
avanzare fino a un retry riuscito.
|
||||
|
||||
**Evidence receipt** — La traccia minima di una consultazione disponibile conservata
|
||||
nella sessione: stage semantico, purpose, generazione interrogata e identificatori delle
|
||||
Evidence restituite. Non duplica il contenuto delle Evidence.
|
||||
|
||||
**Curated Evidence** — Una o più Evidence Unit ristrutturate a partire da una Source
|
||||
Evidence e conservate nel repository del workspace come proposte per la revisione
|
||||
umana. Git conserva la versione precedente e rende visibile ogni modifica; una Curated
|
||||
Evidence non è ancora contenuto autorevole del runtime.
|
||||
|
||||
**Published Evidence** — Le Curated Evidence valide appartenenti alla revisione attiva
|
||||
del workspace e alla generazione Evidence pubblicata. L'approvazione umana precede
|
||||
l'attivazione, ma non viene duplicata come stato nel manifest.
|
||||
|
||||
**Evidence Index** — La proiezione ricercabile e ricostruibile delle Published Evidence.
|
||||
Accelera il recupero delle informazioni, ma non è una fonte di verità.
|
||||
|
||||
**Evidence preparation** — Il processo di authoring che trasforma Source Evidence in
|
||||
Curated Evidence mediante estrazione e normalizzazione deterministiche, una singola
|
||||
ristrutturazione assistita dal modello e una validazione finale deterministica. Nella
|
||||
prima versione accetta Markdown o testo UTF-8 e non acquisisce automaticamente il
|
||||
contenuto di URL o documenti esterni. Prepara l'intero insieme delle modifiche in
|
||||
un'area temporanea e lo applica atomicamente soltanto se tutti gli output sono validi;
|
||||
non ritenta automaticamente una chiamata al modello fallita.
|
||||
|
||||
**Supporting excerpt** — Un breve estratto presente nel Source Evidence che sostiene
|
||||
una Evidence Unit. Il sistema ne verifica deterministicamente la presenza dopo la
|
||||
normalizzazione meccanica; il curatore resta responsabile di verificarne la sufficienza
|
||||
semantica.
|
||||
|
||||
**Evidence resolution** — L'operazione esplicita con cui un curatore ritira una
|
||||
Evidence Unit oppure la ricollega a un Source Evidence esistente. Aggiorna documento e
|
||||
manifest insieme, lascia un diff Git revisionabile e non pubblica né crea commit.
|
||||
|
||||
**Review item** — Un blocco di revisione descritto da codice stabile, messaggio umano e
|
||||
campo opzionale. Finché viene mantenuto nell'Evidence Unit, ne impedisce la
|
||||
pubblicazione; la sua storia è conservata da Git, non da uno stato interno all'item.
|
||||
|
||||
**Retirement candidate** — Una Curated Evidence che il Source Evidence esistente non
|
||||
sostiene più. Rimane visibile con un Review item e blocca la pubblicazione finché il
|
||||
curatore non la elimina oppure la rende nuovamente coerente con il sorgente.
|
||||
|
||||
**Evidence evaluation set** — Un piccolo insieme versionato di domande rappresentative
|
||||
e relativi risultati attesi. La baseline è accettabile quando ogni domanda recupera
|
||||
almeno un risultato atteso nei primi dieci risultati della fusione RRF; il risultato
|
||||
nei primi cinque è informativo. Comprende almeno un caso lessicale, uno semantico e uno
|
||||
misto e conserva, a fini diagnostici, le posizioni dense, BM25 e fused.
|
||||
|
||||
**Candidate Evidence Generation** — Una generazione completa dell'Evidence Index che
|
||||
può essere valutata ma non è ancora visibile alle sessioni. Diventa attiva soltanto se
|
||||
supera l'Evidence evaluation set.
|
||||
|
||||
**Evidence manifest** — Il file versionato e gestito dal sistema che collega ogni
|
||||
Source Evidence al suo hash e alle Evidence Unit derivate. Conserva gli identificatori
|
||||
stabili, permette l'elaborazione incrementale e segnala le unità rimaste orfane senza
|
||||
cancellarle automaticamente.
|
||||
|
||||
**Orphaned Evidence Unit** — Una Curated Evidence il cui Source Evidence non esiste più.
|
||||
Rimane disponibile per la revisione, ma blocca la pubblicazione finché non viene
|
||||
eliminata, ricollegata oppure ne viene ripristinato il sorgente.
|
||||
|
||||
**Evidence Fragment** — Una proiezione ricercabile di una sezione semanticamente
|
||||
coerente di una Published Evidence. La divisione segue intestazioni e confini di
|
||||
paragrafo; formule, coppie valore/significato, mapping, regole e URL non vengono mai
|
||||
tagliati. Il testo completo reso per il frammento usa il solo limite esistente
|
||||
`max_chunk_chars`, pari per default a 4.000 caratteri; un elemento atomico troppo grande
|
||||
produce un Review item bloccante. Qdrant indicizza i frammenti, mentre l'Evidence Module
|
||||
li raggruppa per Evidence Unit.
|
||||
|
||||
**Evidence Result** — La rappresentazione di una singola Evidence Unit restituita dalla
|
||||
ricerca con metadati, migliori estratti, provenienza e riferimento al documento completo.
|
||||
|
||||
**Hybrid Evidence retrieval** — La ricerca che combina in Qdrant una graduatoria
|
||||
semantica dense e una graduatoria lessicale BM25 sparse mediante Reciprocal Rank
|
||||
Fusion. I metadati tipizzati restringono o orientano i risultati senza creare una
|
||||
collezione separata per ogni Evidence kind.
|
||||
|
||||
**Evidence query text** — La rappresentazione deterministica condivisa dalla ricerca
|
||||
dense e BM25: domanda originale, concetti, tabelle e colonne in ordine fisso. I campi
|
||||
vuoti sono omessi; domanda e contesto ricevono soltanto normalizzazione Unicode NFC,
|
||||
conversione degli a-capo e rimozione degli spazi esterni. Gli elementi contestuali sono
|
||||
poi deduplicati e ordinati senza conversione delle maiuscole, mentre punteggiatura e
|
||||
spazi interni della domanda non vengono riscritti.
|
||||
|
||||
**Additive BM25 upgrade** — L'estensione non distruttiva della collezione semantica di
|
||||
un workspace che conserva il vettore dense predefinito e aggiunge il solo vettore
|
||||
sparse `bm25`. Soltanto gli Evidence Fragment ricevono valori BM25; Schema e Memory
|
||||
mantengono invariati dati e ricerca dense.
|
||||
|
||||
**Formula proposal** — Una formula individuata durante una sessione e conservata come
|
||||
artefatto della sessione. Non diventa Published Evidence finché non viene importata,
|
||||
revisionata e approvata nel repository del workspace.
|
||||
|
||||
**Fail-closed Evidence retrieval** — Il comportamento per cui un indice assente,
|
||||
incompatibile o non aggiornato produce nessuna Evidence e un avviso esplicito. Il
|
||||
workflow può continuare, ma non usa mai silenziosamente contenuti di una revisione
|
||||
precedente o di un altro workspace.
|
||||
|
||||
## Configurazione dei modelli
|
||||
|
||||
**Workspace Descriptor** — La dichiarazione versionata dell'identità e dello scope del
|
||||
Workspace Database e delle Evidence di un workspace. Non definisce modelli, selezioni di
|
||||
modello o Database Binding specifiche di un'installazione.
|
||||
|
||||
**Installation Model Catalog** — L'insieme dichiarativo, proprio di un'installazione, dei
|
||||
modelli disponibili, dei loro Model Usage e dei relativi default. È l'unica autorità per i
|
||||
modelli di sessione, generazione dei metadati ed embedding e non appartiene a un workspace.
|
||||
_Avoid_: Model Catalog, Metadata Generation Model Configuration
|
||||
|
||||
**Model Usage** — Lo scopo per cui un modello dell'Installation Model Catalog può essere
|
||||
usato: `session`, `metadata_generation` oppure `embedding`. L'ammissibilità e il default
|
||||
dipendono dall'uso, non dal workspace.
|
||||
|
||||
**Model Selection** — La scelta runtime, a livello di installazione, di un modello del
|
||||
catalogo per uno specifico Model Usage. Riferisce l'identità canonica del modello senza
|
||||
ridefinirne provider, endpoint o capacità.
|
||||
|
||||
**Model Runtime Projection** — La rappresentazione derivata e non autoritativa
|
||||
dell'Installation Model Catalog richiesta da uno specifico runtime. Può essere rigenerata
|
||||
integralmente dalla configurazione dell'installazione.
|
||||
|
||||
## Catalogo dei metadati
|
||||
|
||||
**Workspace Database** — Il database che appartiene a un solo workspace e non può essere
|
||||
condiviso con altri workspace; un workspace può averne al massimo uno. È considerato nella
|
||||
coppia composta dal database PostgreSQL e da un solo schema: tutte le tabelle, le colonne e
|
||||
le relazioni catalogate appartengono a quello schema. Il Metadata Catalog conserva
|
||||
l'associazione, ma non crea né possiede l'identità del workspace.
|
||||
|
||||
**Database Binding** — La configurazione specifica di un'installazione che seleziona un
|
||||
trasporto e fornisce i riferimenti necessari a raggiungere un Workspace Database. Non è una
|
||||
seconda identità del database e non viene condivisa automaticamente fra installazioni.
|
||||
|
||||
**Thoth REST Connector** — Il trasporto REST tipizzato con cui ThothII interroga ed
|
||||
introspeziona un Workspace Database attraverso il contratto RPC DWH supportato. Non è un
|
||||
client configurabile per API REST arbitrarie.
|
||||
|
||||
**Orphaned Workspace Database** — Un Workspace Database il cui workspace non è più presente
|
||||
nel catalogo autorevole. Rimane conservato per il recupero amministrativo, ma non può essere
|
||||
usato dal workflow finché non viene riassegnato a un workspace esistente.
|
||||
|
||||
**Metadata Catalog** — Il contesto amministrativo che raccoglie e cura i metadati di un
|
||||
Workspace Database. Non definisce quali elementi partecipano al workflow SQL.
|
||||
|
||||
**Database Profile** — L'insieme curato di scope, descrizioni e metadati semantici
|
||||
associato a un Workspace Database.
|
||||
|
||||
**Physical Table** — Una tabella osservata nello schema esterno di un Workspace Database.
|
||||
La sua identità e il suo nome appartengono al database esterno, non al Metadata Catalog.
|
||||
|
||||
**Catalog Table** — La rappresentazione persistita di una Physical Table nel Metadata Catalog.
|
||||
La sua appartenenza e identità fisica derivano dall'introspezione: non può essere creata o
|
||||
rinominata manualmente, ma può essere rimossa tramite Catalog Metadata Cleanup.
|
||||
_Avoid_: SqlTable, managed table
|
||||
|
||||
**Physical Column** — Una colonna osservata in una Physical Table, inclusi nome, posizione,
|
||||
tipo e appartenenza a chiavi dichiarate. La sua identità e i suoi fatti strutturali appartengono
|
||||
al database esterno.
|
||||
|
||||
**Catalog Column** — La rappresentazione persistita di una Physical Column nel Metadata Catalog.
|
||||
I fatti osservati sono governati dalla sincronizzazione; Description e Generated Description
|
||||
sono metadati amministrativi modificabili e la rappresentazione può essere rimossa tramite
|
||||
Catalog Metadata Cleanup.
|
||||
_Avoid_: SqlColumn, managed column
|
||||
|
||||
**Physical Relationship** — Un vincolo foreign key dichiarato nel database esterno. La sua
|
||||
identità comprende il vincolo e la sequenza ordinata delle coppie di colonne che lo compongono.
|
||||
|
||||
**Catalog Relationship** — La rappresentazione persistita di una Physical Relationship nel
|
||||
Metadata Catalog. Non è creata o modificata manualmente, ma può essere rimossa tramite Catalog
|
||||
Metadata Cleanup.
|
||||
_Avoid_: denormalized FK, relationship string
|
||||
|
||||
**Logical Relationship** — Una relazione modificabile fra due Catalog Column che non corrisponde
|
||||
necessariamente a un vincolo fisico. Può essere Generated o Manual e rimane distinta dalla Catalog
|
||||
Relationship osservata nel database.
|
||||
|
||||
**Generated Relationship** — Una Logical Relationship ricavata dai nomi delle colonne, dalle
|
||||
primary key e dalla compatibilità dei tipi mediante regole deterministiche, senza LLM, embedding o
|
||||
campionamento dei dati. Una ricostruzione non riattiva una Generated Relationship cancellata
|
||||
logicamente, ma può ricrearne una cancellata fisicamente.
|
||||
|
||||
**Manual Relationship** — Una Logical Relationship aggiunta dall'utente. La ricostruzione delle
|
||||
Generated Relationship non la modifica.
|
||||
|
||||
**Logical Relationship Deletion** — L'esclusione persistente di una Logical Relationship che ne
|
||||
conserva l'identità per impedirne la ricreazione automatica finché esistono entrambe le Catalog
|
||||
Column alle quali è collegata.
|
||||
|
||||
**Permanent Relationship Deletion** — La rimozione completa di una Logical Relationship. Una
|
||||
ricostruzione successiva può ricrearla quando soddisfa nuovamente le regole di inferenza. Anche il
|
||||
cleanup distruttivo di una tabella o colonna endpoint rimuove permanentemente le relative esclusioni.
|
||||
|
||||
**Relationship Reconstruction** — L'operazione amministrativa esplicita che scopre e aggiunge le
|
||||
Generated Relationship mancanti. Conserva le Manual Relationship e le relationship già presenti e
|
||||
non riattiva quelle cancellate logicamente.
|
||||
|
||||
**Relationship Restore** — La riattivazione esplicita di una Logical Relationship cancellata
|
||||
logicamente.
|
||||
|
||||
**Effective Relationship Map** — La vista unificata delle Catalog Relationship fisiche e delle
|
||||
Logical Relationship, con origine e stato espliciti. È l'interfaccia usata dall'amministrazione e
|
||||
dalla comprensione dello schema, non un ulteriore modello persistito.
|
||||
|
||||
**Effective Relationship Snapshot** — La proiezione runtime immutabile delle relationship attive
|
||||
contenute nell'Effective Relationship Map. È derivata dal Metadata Catalog per una singola sessione
|
||||
e viene eliminata insieme alla relativa configurazione runtime.
|
||||
|
||||
**Description** — Il testo curato e consolidato che descrive una Catalog Table o Catalog Column
|
||||
per gli usi downstream.
|
||||
|
||||
**Generated Description** — Una proposta modificabile sottoposta a revisione umana prima di
|
||||
essere consolidata come Description. Rimane distinta dal commento osservato nel database.
|
||||
_Avoid_: generated comment, source comment
|
||||
|
||||
**Description Consolidation** — L'azione amministrativa esplicita che copia la Generated
|
||||
Description di Catalog Table o Catalog Column selezionate nella relativa Description. Opera sulla
|
||||
selezione corrente, conserva la Generated Description e non modifica il commento osservato o il
|
||||
database esterno.
|
||||
|
||||
**Table Synchronization** — La riconciliazione esplicita che rende le Catalog Table di un
|
||||
Workspace Database uguali alle Physical Table osservate: crea quelle nuove, aggiorna i metadati
|
||||
di origine ed elimina definitivamente quelle assenti. Non modifica mai il database esterno.
|
||||
_Avoid_: table import
|
||||
|
||||
**Schema Synchronization** — La riconciliazione esplicita e autorevole di tabelle, colonne e
|
||||
Catalog Relationship di un Workspace Database. Può operare su uno scope specifico oppure su
|
||||
un unico snapshot completo tramite Synchronize All.
|
||||
|
||||
**Catalog Sync Run** — L'esecuzione durevole in background di una Schema Synchronization, con
|
||||
scope, stato, avanzamento e log propri. Al massimo un run per Workspace Database può essere attivo.
|
||||
|
||||
**Description Generation Run** — L'esecuzione asincrona e sequenziale che usa il modello scelto
|
||||
per produrre Generated Description di Catalog Table o Catalog Column. Al massimo una run è attiva
|
||||
nell'intera installazione e ogni risultato valido viene salvato appena disponibile. Dopo
|
||||
un'interruzione il recupero è manuale tramite una nuova generazione dei soli elementi mancanti.
|
||||
|
||||
**Description Generation Event** — Una riga testuale ordinata che registra avanzamento, risultato
|
||||
o errore di una Description Generation Run e alimenta il log visibile all'amministratore.
|
||||
|
||||
**Non-generatable Description** — L'esito valido con cui il modello dichiara di non disporre di
|
||||
informazioni sufficienti per descrivere il target. Produce una Generated Description standard
|
||||
nella lingua del workspace e non rappresenta un timeout, un errore del provider o una risposta
|
||||
non valida.
|
||||
|
||||
**Description Generation Unlock** — Il recupero amministrativo che marca come interrotta una
|
||||
Description Generation Run registrata come attiva quando il backend non ha alcun processo di
|
||||
generazione vivo. Non è un meccanismo di lock distribuito.
|
||||
|
||||
**Catalog Metadata Cleanup** — La rimozione amministrativa esplicita di Catalog Table, Catalog
|
||||
Column o Catalog Relationship selezionate. Non modifica il Workspace Database, la Database Binding
|
||||
o i segreti, e può lasciare il Metadata Catalog intenzionalmente incompleto fino alla prossima
|
||||
Schema Synchronization.
|
||||
|
||||
**Catalog Freshness** — La corrispondenza fra uno scope sincronizzato e la versione corrente
|
||||
della Database Binding. Uno scope rimane consultabile ma è stale finché non viene sincronizzato
|
||||
con la binding corrente.
|
||||
|
||||
**Catalog Metadata** — I campi mutabili che descrivono database, tabelle, colonne e relazioni,
|
||||
distinti dai fatti strutturali governati dalla sincronizzazione. Possono essere popolati dall'AI,
|
||||
da un'importazione o da una modifica amministrativa senza cambiare il database esterno.
|
||||
|
||||
**Model Completion Helper** — Il processo Python interno ed effimero che esegue una singola
|
||||
richiesta LiteLLM per conto del backend. Non è un servizio HTTP, non possiede il lifecycle della
|
||||
Description Generation Run e non è una CLI esposta agli utenti.
|
||||
|
||||
**Catalog Sample** — Un input transitorio composto da un massimo di cinque righe e da valori di
|
||||
esempio bounded di una Catalog Table per la generazione delle descrizioni. Può contenere valori
|
||||
reali oppure sintetici in base alla Source Value Disclosure Decision; non viene persistito e non
|
||||
diventa Catalog Metadata.
|
||||
|
||||
**Sensitive Data Flag** — La classificazione binaria umana applicata a una Catalog Column. Può
|
||||
essere impostata liberamente dall'amministratore anche in contrasto con una valutazione automatica.
|
||||
|
||||
**Local Sensitivity Assessment** — La valutazione locale, non autoritativa e priva di LLM di una
|
||||
Catalog Column, basata su metadati e contenuto sorgente, con esito `sensitive`, `non_sensitive`
|
||||
oppure `unknown`.
|
||||
_Avoid_: AI suggestion, automatic flag
|
||||
|
||||
**Local NER Detector** — Il componente NLP opzionale e CPU-only che esamina soltanto testo ancora
|
||||
ambiguo e restituisce evidenze al Local Sensitivity Assessment. Non decide lo stato della colonna,
|
||||
non usa un LLM generativo e non persiste valori sorgente.
|
||||
_Avoid_: AI classifier, local LLM fallback
|
||||
|
||||
**Model Data Boundary** — La qualificazione amministrativa di un modello come `internal` oppure
|
||||
`external` rispetto al confine entro cui i valori sorgente possono essere comunicati.
|
||||
_Avoid_: local model, remote model
|
||||
|
||||
**Source Value Disclosure Decision** — L'unica decisione effettiva che stabilisce se un modello
|
||||
riceve valori sorgente reali oppure sostituti sintetici, combinando Model Data Boundary e Sensitive
|
||||
Data Flag.
|
||||
_Avoid_: sample filter, export flag
|
||||
|
||||
**Sensitive Data Policy** — L'insieme versionato di regole locali generali e specifiche che produce
|
||||
una Local Sensitivity Assessment. Un singolo riscontro blocca l'intera colonna e qualsiasi valore
|
||||
testuale più lungo di 500 caratteri rende sensibile la colonna.
|
||||
_Avoid_: PII filter, sample filter
|
||||
|
||||
**Sensitivity Analysis Run** — Il tentativo amministrativo esplicito e tracciato che valuta una
|
||||
selezione di colonne mediante la Sensitive Data Policy. Conserva stato, copertura e conteggi
|
||||
aggregati, ma non valori sorgente né esiti per colonna.
|
||||
_Avoid_: Sensitive Data Suggestion Run, AI analysis
|
||||
|
||||
**Sensitivity Review Draft** — La proposta transitoria che associa alle colonne selezionate una
|
||||
Local Sensitivity Assessment e le relative evidenze sanificate. Non modifica il Sensitive Data Flag
|
||||
finché l'amministratore non salva le proprie decisioni e viene scartata al reload.
|
||||
_Avoid_: automatic flag
|
||||
|
||||
**Sensitivity Analysis Event** — Una riga testuale ordinata e sanificata che registra l'avvio,
|
||||
l'esito o l'errore di una Sensitivity Analysis Run senza conservare contenuti sorgente, output grezzi
|
||||
del detector o proposte per colonna.
|
||||
_Avoid_: Sensitive Data Suggestion Event
|
||||
|
||||
**Introspection Capability** — Una categoria di struttura fisica che una Database Binding
|
||||
può osservare, come tabelle, colonne, relazioni, indici o enum. Una capability non disponibile
|
||||
è distinta da una capability osservata che non ha restituito elementi.
|
||||
@@ -1,363 +0,0 @@
|
||||
---
|
||||
name: ThothII
|
||||
description: "A calm, precise clinical analytics workbench for traceable and reviewable SQL workflows."
|
||||
colors:
|
||||
instrument-red: "oklch(55.87% 0.1881 23.2)"
|
||||
instrument-red-hover: "oklch(50.95% 0.1812 24.1)"
|
||||
porcelain-background: "oklch(99.18% 0.0011 17.2)"
|
||||
porcelain-card: "oklch(99.85% 0.0006 17.2)"
|
||||
warm-surface: "oklch(97.09% 0.0011 17.2)"
|
||||
sunken-surface: "oklch(94.08% 0.0011 17.2)"
|
||||
warm-graphite: "oklch(26.78% 0.0097 355.6)"
|
||||
muted-graphite: "oklch(51.33% 0.0088 345.6)"
|
||||
quiet-border: "oklch(90.93% 0.0035 354.7)"
|
||||
success-mint: "oklch(75.77% 0.1581 165)"
|
||||
navigation-active: "oklch(92.5% 0.052 23.2)"
|
||||
navigation-active-hover: "oklch(89.5% 0.071 23.2)"
|
||||
navigation-active-foreground: "oklch(36.5% 0.11 23.2)"
|
||||
navigation-active-border: "oklch(60% 0.135 23.2)"
|
||||
warning-amber: "oklch(85.23% 0.1386 78.9)"
|
||||
information-blue: "oklch(70.35% 0.1128 221.3)"
|
||||
typography:
|
||||
display:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "3rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.03
|
||||
letterSpacing: "-0.025em"
|
||||
headline:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "1.875rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.15
|
||||
letterSpacing: "-0.015em"
|
||||
title:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "1.2rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "-0.01em"
|
||||
body:
|
||||
fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "0.9375rem"
|
||||
fontWeight: 400
|
||||
lineHeight: 1.65
|
||||
letterSpacing: "normal"
|
||||
control:
|
||||
fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "0.875rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "0.005em"
|
||||
label:
|
||||
fontFamily: "ui-monospace, SF Mono, Cascadia Code, Menlo, Consolas, monospace"
|
||||
fontSize: "0.6875rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "0.06em"
|
||||
rounded:
|
||||
xs: "4px"
|
||||
sm: "6px"
|
||||
md: "8px"
|
||||
lg: "12px"
|
||||
xl: "16px"
|
||||
full: "9999px"
|
||||
spacing:
|
||||
xs: "4px"
|
||||
sm: "8px"
|
||||
md: "16px"
|
||||
lg: "24px"
|
||||
xl: "32px"
|
||||
components:
|
||||
button-primary:
|
||||
backgroundColor: "{colors.instrument-red}"
|
||||
textColor: "{colors.porcelain-background}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 14px"
|
||||
height: "32px"
|
||||
button-primary-hover:
|
||||
backgroundColor: "{colors.instrument-red-hover}"
|
||||
textColor: "{colors.porcelain-background}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 14px"
|
||||
height: "32px"
|
||||
button-secondary:
|
||||
backgroundColor: "{colors.porcelain-card}"
|
||||
textColor: "{colors.warm-graphite}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 14px"
|
||||
height: "32px"
|
||||
input-default:
|
||||
backgroundColor: "{colors.porcelain-background}"
|
||||
textColor: "{colors.warm-graphite}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 12px"
|
||||
height: "40px"
|
||||
card-default:
|
||||
backgroundColor: "{colors.porcelain-card}"
|
||||
textColor: "{colors.warm-graphite}"
|
||||
rounded: "{rounded.lg}"
|
||||
padding: "16px"
|
||||
badge-primary:
|
||||
backgroundColor: "{colors.instrument-red}"
|
||||
textColor: "{colors.porcelain-background}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.sm}"
|
||||
padding: "2px 8px"
|
||||
height: "20px"
|
||||
---
|
||||
|
||||
# Design System: ThothII
|
||||
|
||||
## Overview
|
||||
|
||||
**Creative North Star: "The Clinical Workbench"**
|
||||
|
||||
ThothII should feel like a well-kept clinical workbench: warm enough for sustained reading, exact
|
||||
enough for consequential review, and quiet enough that evidence, state, and decisions remain in the
|
||||
foreground. The visual system is calm, precise, and trustworthy. It uses familiar product patterns,
|
||||
restrained color, and deliberate density instead of decorative spectacle.
|
||||
|
||||
The primary physical scene is an analyst reviewing persisted evidence and SQL on a large monitor in
|
||||
a well-lit working environment. This makes the warm light theme the default. The supported dark
|
||||
theme serves lower-light work without becoming a separate neon aesthetic. Both themes preserve the
|
||||
same hierarchy and semantic roles.
|
||||
|
||||
The system rejects generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long
|
||||
choreographed transitions, and effects that compete with the analytical task. Controls should feel
|
||||
disciplined and tactile, never playful, sluggish, or visually unstable.
|
||||
|
||||
**Key Characteristics:**
|
||||
|
||||
- Warm, restrained surfaces with one scarce red accent.
|
||||
- Editorial headings paired with highly legible operational body text.
|
||||
- Dense information organized through hierarchy, rhythm, and progressive disclosure.
|
||||
- Persisted artifacts and reviewer decisions presented as the visual source of truth.
|
||||
- Fast state feedback with reduced-motion parity.
|
||||
|
||||
**The Workbench Rule.** Every visual element must support inspection, action, state, or provenance.
|
||||
Decoration without an operational purpose is forbidden.
|
||||
|
||||
**The Persisted Truth Rule.** Persisted artifacts and reviewer decisions receive stronger hierarchy
|
||||
than transient model narration.
|
||||
|
||||
**The Density with Rhythm Rule.** Preserve information density, but vary spacing between groups so
|
||||
users can scan structure without adding nested containers.
|
||||
|
||||
## Colors
|
||||
|
||||
The palette combines warm porcelain surfaces, warm graphite text, and an instrument red used only
|
||||
for action, focus, and important state. OKLCH values in the frontmatter are normative because the
|
||||
frontend uses OKLCH tokens directly.
|
||||
|
||||
### Primary
|
||||
|
||||
- **Instrument Red** (`instrument-red`): primary actions, focus identity, and destructive meaning
|
||||
where the context already makes the action explicit.
|
||||
- **Instrument Red Pressed** (`instrument-red-hover`): hover and active emphasis for the primary
|
||||
action family.
|
||||
|
||||
### Neutral
|
||||
|
||||
- **Porcelain Background** (`porcelain-background`): the main canvas.
|
||||
- **Porcelain Card** (`porcelain-card`): lifted panels, cards, and popovers.
|
||||
- **Warm Surface** (`warm-surface`): sidebars, secondary controls, and muted regions.
|
||||
- **Sunken Surface** (`sunken-surface`): selected rows, quiet emphasis, and inset regions.
|
||||
- **Warm Graphite** (`warm-graphite`): primary text and high-confidence labels.
|
||||
- **Muted Graphite** (`muted-graphite`): descriptions, timestamps, and secondary metadata.
|
||||
- **Quiet Border** (`quiet-border`): structural boundaries, input outlines, and dividers.
|
||||
|
||||
### Semantic
|
||||
|
||||
- **Success Mint** (`success-mint`): completed and ready states.
|
||||
- **Navigation Active** (`navigation-active`): the one application surface currently in the
|
||||
foreground. It shares Instrument Red's hue but uses a lighter, lower-chroma fill, so location is
|
||||
visible without carrying the full weight of a primary action.
|
||||
- **Warning Amber** (`warning-amber`): waiting, attention, and in-progress states.
|
||||
- **Information Blue** (`information-blue`): informational state when red would imply action.
|
||||
|
||||
The dark theme keeps the same semantic mapping with neutral near-black surfaces and a slightly
|
||||
lighter red accent. Do not introduce a second visual identity for dark mode.
|
||||
|
||||
**The One Voice Rule.** Instrument Red should occupy no more than roughly ten percent of a screen.
|
||||
Its rarity is what makes it authoritative.
|
||||
|
||||
**The State Has a Name Rule.** Success, warning, information, and destructive colors are reserved
|
||||
for their named states. Color is never the only state indicator.
|
||||
|
||||
## Typography
|
||||
|
||||
**Display Font:** Fraunces, with Source Serif Pro, Georgia, and Times New Roman fallbacks
|
||||
**Body Font:** Manrope, with native system sans-serif fallbacks
|
||||
**Label/Mono Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks
|
||||
|
||||
**Character:** Fraunces gives persisted artifacts and key headings editorial authority. Manrope
|
||||
keeps dense controls and prose calm and readable. The mono register separates machine identity,
|
||||
metadata, SQL, identifiers, and micro-labels from natural-language content.
|
||||
|
||||
### Hierarchy
|
||||
|
||||
- **Display** (600, `3rem`, `1.03`): authentication and exceptional page-level statements only.
|
||||
- **Headline** (600, `1.875rem`, `1.15`): major page or artifact titles.
|
||||
- **Title** (600, `1.2rem`, `1.25`): panel and document section hierarchy.
|
||||
- **Body** (400, `0.9375rem`, `1.65`): operational prose, with a target line length of 65 to 75
|
||||
characters where the surface controls width.
|
||||
- **Control** (600, `0.875rem`, `1.25`): buttons, inputs, tabs, and compact actions.
|
||||
- **Label** (600, `0.6875rem`, `0.06em` tracking): uppercase micro-labels, state metadata, and panel
|
||||
headers. Labels use the mono family.
|
||||
|
||||
Typography uses fixed sizes. Responsive changes happen at structural breakpoints, not through fluid
|
||||
type scaling. Numeric data and identifiers use tabular numerals where comparison matters.
|
||||
|
||||
**The Three Registers Rule.** Serif means authority, sans means interaction and reading, mono means
|
||||
machine identity. Do not exchange these roles for novelty.
|
||||
|
||||
**The Read Once Rule.** A heading, label, and body must be distinguishable on first glance through
|
||||
size and weight. Do not repeat headings in explanatory copy.
|
||||
|
||||
## Elevation
|
||||
|
||||
The system is flat by default and layered when necessary. Borders mark structure. Warm, diffuse
|
||||
shadows mark actual elevation for popovers, dialogs, and selected containers. Tonal layering should
|
||||
solve most hierarchy before a shadow is introduced.
|
||||
|
||||
### Shadow Vocabulary
|
||||
|
||||
- **Contact Shadow** (`--shadow-xs`): a one-pixel contact shadow for controls and code blocks.
|
||||
- **Panel Shadow** (`--shadow-sm`): a small two-stage shadow for cards that need separation from the
|
||||
canvas.
|
||||
- **Overlay Shadow** (`--shadow-md`): a broad, low-opacity shadow for dialogs and floating layers.
|
||||
|
||||
Focus uses an explicit three-pixel ring. Waiting-for-input state may use a success-tinted ring, but
|
||||
must retain a textual or structural cue. Motion for button state changes lasts `140ms` with
|
||||
`cubic-bezier(0.22, 1, 0.36, 1)`. Dialog transitions last `100ms`. Activity pulses may run at
|
||||
`1.5s`, and must be disabled under `prefers-reduced-motion`.
|
||||
|
||||
**The Flat by Default Rule.** A resting surface has no shadow unless it is physically above another
|
||||
surface. If every panel floats, none of them has hierarchy.
|
||||
|
||||
**The Borders Structure, Shadows Elevate Rule.** Never use shadow as a substitute for grouping or a
|
||||
border as a decorative accent.
|
||||
|
||||
## Components
|
||||
|
||||
Components are familiar, compact, and state-complete. Every interactive primitive must define
|
||||
default, hover, focus, active, disabled, loading, and error behavior where those states apply.
|
||||
|
||||
### Buttons
|
||||
|
||||
- **Shape:** gently curved rectangle (`8px`) with a one-pixel transparent or structural border.
|
||||
- **Primary:** Instrument Red, porcelain text, `32px` default height, and `14px` horizontal padding.
|
||||
- **Hover / Focus:** shift to Instrument Red Pressed; show a three-pixel focus ring at 25 percent
|
||||
opacity. Active state scales to `0.97` for `140ms` and removes elevation.
|
||||
- **Secondary / Outline:** porcelain card surface, Quiet Border, Warm Graphite text, and a Warm
|
||||
Surface hover.
|
||||
- **Ghost:** transparent at rest, Warm Surface on hover. Use only where surrounding structure makes
|
||||
the hit target obvious.
|
||||
|
||||
### Badges and Status Indicators
|
||||
|
||||
- **Style:** compact (`20px` height), gently curved (`6px`), and semibold.
|
||||
- **State:** pair semantic color with text, icon, or position. A colored dot alone is insufficient
|
||||
when the state affects workflow decisions.
|
||||
|
||||
### Cards and Containers
|
||||
|
||||
- **Corner Style:** softly rounded (`12px`), with `16px` default internal padding.
|
||||
- **Background:** Porcelain Card over Porcelain Background or Warm Surface.
|
||||
- **Shadow Strategy:** Panel Shadow only when the card must read as elevated.
|
||||
- **Border:** one-pixel Quiet Border at partial opacity.
|
||||
- **Nesting:** nested cards are forbidden. Use headings, dividers, spacing, or tonal regions.
|
||||
|
||||
### Inputs and Fields
|
||||
|
||||
- **Style:** `40px` height, `8px` corners, Porcelain Background, Quiet Border, and Manrope body text.
|
||||
- **Focus:** three-pixel Instrument Red ring with a clear border shift.
|
||||
- **Error / Disabled:** errors combine destructive color with explanatory text; disabled controls
|
||||
retain readable contrast and use 50 percent opacity.
|
||||
- **Metadata catalog model:** Database Management keeps one compact, installation-level
|
||||
metadata-generation LLM selector in the application header. The selection persists across
|
||||
database, table, column, and relationship views; when no usable profile is configured, the
|
||||
disabled control explains: “No metadata-generation LLM model is configured for this installation.”
|
||||
|
||||
### Navigation
|
||||
|
||||
- **Style:** compact session rows use `8px` corners and restrained vertical padding.
|
||||
- **Default / Hover / Active:** porcelain at rest, Sunken Surface on hover, and a muted Navigation
|
||||
Active red with a defined border when current. Exactly one top-level navigation control is current.
|
||||
- **Administrative controls:** the admin-only Administration accordion groups Database management,
|
||||
a structural divider, Workspace management, and Pi management in that order. Its trigger exposes
|
||||
expanded state and starts collapsed by default, while non-admin users do not receive the accordion
|
||||
or its navigation actions.
|
||||
- **Responsive:** collapse navigation structurally at the application breakpoint. Do not shrink
|
||||
labels into illegibility.
|
||||
|
||||
### Tabs
|
||||
|
||||
- **Shape:** compact label tabs sit on a shared baseline with rounded top corners and a two-pixel
|
||||
lower edge. Inactive labels retain a complete Quiet Border and Porcelain Card surface, so every
|
||||
label reads as a tab before interaction; hover feedback reinforces clickability.
|
||||
- **Current:** the selected tab uses the muted Navigation Active red for its fill, text, and defined border.
|
||||
It must expose `aria-selected`, participate in a labelled `tablist`/`tabpanel`, and be the only
|
||||
tab in the roving keyboard tab order.
|
||||
- **Keyboard:** Left/Right move between adjacent tabs with wrapping; Home/End select the first or
|
||||
last tab.
|
||||
|
||||
### Tooltips
|
||||
|
||||
- **Row actions:** icon-action tooltips open three pixels below the trigger and align to its trailing
|
||||
edge, so they never cover the icon row. They use a dark slate surface, porcelain text, and a
|
||||
defined border rather than the light popover treatment.
|
||||
- **Interaction:** tooltip layers never receive pointer events. They appear on hover and keyboard
|
||||
focus with a short ease-out transition, while the icon button keeps its complete accessible name.
|
||||
- **Scope:** this treatment is shared by database, table, column, and relationship row actions.
|
||||
Toolbar and navigation hints may use separate collision-aware placement.
|
||||
|
||||
### Curated Evidence Documents
|
||||
|
||||
Curated evidence follows a fixed reading order: title, compact type and purpose summary, scope,
|
||||
typed content, supporting excerpts, review items, then collapsed technical provenance. Machine
|
||||
metadata stays in invisible comments so GitHub Preview shows only the reviewable document.
|
||||
|
||||
`applies_to` is rendered as “Ambito di applicazione” with separate bullet lists for concepts,
|
||||
tables, and columns. Enum values also use lists. Tables are forbidden for metadata, scope, or any
|
||||
one-dimensional collection; reserve tables for genuinely two-dimensional datasets. Long machine
|
||||
identifiers use inline code. SQL uses fenced code. Supporting excerpts use blockquotes.
|
||||
|
||||
**The Review Surface Rule.** The visible Markdown must be readable without understanding the
|
||||
machine contract. Technical metadata belongs in progressive disclosure, not above the title.
|
||||
|
||||
## Do's and Don'ts
|
||||
|
||||
### Do:
|
||||
|
||||
- **Do** make every state change unmistakable without interrupting flow.
|
||||
- **Do** use Instrument Red only for primary action, current selection, focus identity, or explicit
|
||||
destructive meaning.
|
||||
- **Do** preserve information density with headings, rhythm, and progressive disclosure.
|
||||
- **Do** keep keyboard focus explicit and pair color with text, shape, icon, or position.
|
||||
- **Do** respect `prefers-reduced-motion` while preserving immediate non-kinetic feedback.
|
||||
- **Do** use English for interface chrome and the workspace language for persisted document content.
|
||||
- **Do** render curated metadata and scope as Markdown prose or lists, never as a frontmatter table.
|
||||
- **Do** break long curated rules into paragraphs, labelled subsections, and lists at existing
|
||||
punctuation boundaries while preserving the exact canonical text for machines.
|
||||
|
||||
### Don't:
|
||||
|
||||
- **Don't** add generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long
|
||||
choreographed transitions, or effects that compete with the analytical task.
|
||||
- **Don't** make controls feel playful, sluggish, or visually unstable.
|
||||
- **Don't** use gradient text, decorative glassmorphism, or full-saturation accents on inactive
|
||||
states.
|
||||
- **Don't** use a colored side stripe greater than one pixel on cards, callouts, list items, or
|
||||
blockquotes. Use a full border, tonal background, icon, or heading instead.
|
||||
- **Don't** nest cards or wrap every section in a container.
|
||||
- **Don't** use a modal before exhausting inline or progressive alternatives.
|
||||
- **Don't** use tables for `applies_to`, metadata, enum values, or other one-dimensional content.
|
||||
- **Don't** use color as the sole carrier of success, warning, error, selection, or progress.
|
||||
- **Don't** use display typography for buttons, labels, or data.
|
||||
- **Don't** add em dashes to interface copy. Use commas, colons, semicolons, or parentheses.
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
# Product
|
||||
|
||||
## Register
|
||||
|
||||
product
|
||||
|
||||
## Users
|
||||
|
||||
ThothII serves technical and clinical analysts who build and review datamarts through a guided natural-language-to-SQL workflow. They work in a dense operational interface, often moving repeatedly between model output, persisted artifacts, and human review gates.
|
||||
|
||||
## Product Purpose
|
||||
|
||||
The product turns analytical questions into traceable, reviewable SQL work. It should make model activity, workflow state, and human decisions legible while keeping the analyst in control of every consequential transition.
|
||||
|
||||
## Brand Personality
|
||||
|
||||
Calm, precise, and trustworthy. Interaction feedback should feel as immediate and disciplined as Linear or Raycast, with enough character to acknowledge an action but no decorative spectacle.
|
||||
|
||||
## Anti-references
|
||||
|
||||
Avoid generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long choreographed transitions, and effects that compete with the analytical task. Controls must not feel playful, sluggish, or visually unstable.
|
||||
|
||||
## Design Principles
|
||||
|
||||
1. Make state changes unmistakable without interrupting flow.
|
||||
2. Prefer consistent, familiar controls over novel affordances.
|
||||
3. Keep the persisted workflow and reviewer decisions visually authoritative.
|
||||
4. Use motion only to explain feedback, progress, or spatial relationships.
|
||||
5. Preserve information density while maintaining clear hierarchy.
|
||||
|
||||
## Accessibility & Inclusion
|
||||
|
||||
Keyboard focus must remain explicit, color cannot be the sole carrier of meaning, and motion must respect `prefers-reduced-motion`. Reduced-motion users should retain immediate non-kinetic visual feedback for every action.
|
||||
@@ -1,250 +0,0 @@
|
||||
# ThothII — Project State
|
||||
|
||||
Last updated: 2026-08-31.
|
||||
|
||||
This file is the short operational snapshot. Stable commands and the architecture mental model
|
||||
live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`,
|
||||
`docs/contracts/`, `docs/adr/`, and `docs/evidence.md`. Superseded plans and reports are
|
||||
available from Git history rather than duplicated in the working tree.
|
||||
|
||||
## Current product shape
|
||||
|
||||
ThothII is a human-in-the-loop datamart builder with three independently built layers:
|
||||
|
||||
```text
|
||||
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
|
||||
```
|
||||
|
||||
The harness owns the deterministic eight-phase NL→SQL workflow and all session persistence.
|
||||
The backend remains a process/RPC/SSE bridge for sessions and now also owns an isolated PostgreSQL
|
||||
metadata catalog for administrative database configuration. The frontend renders the review gates
|
||||
and keeps the live transcript in memory. See
|
||||
`docs/architecture/components.md` for the detailed component and data-flow map.
|
||||
|
||||
## Evidence restructuring — accepted
|
||||
|
||||
The evidence restructuring and PSD migration completed real acceptance on 2026-08-25.
|
||||
|
||||
- The curated PSD revision contains 35 approved Evidence units and 60 review items.
|
||||
- The PSD authoring repository publishes all 35 units using Curated unit schema v3. Its table-free
|
||||
presentation uses hidden canonical metadata, wrapping Markdown scope lists, list-based enum
|
||||
values, and collapsed technical provenance. Long domain rules now have a deterministic
|
||||
human-readable presentation while retaining their exact canonical text for vector ingestion.
|
||||
`tht evidence migrate <workspace-root>` performs the deterministic v1/v2 upgrade and older-v3
|
||||
presentation rewrite without model calls. The structured-rule PSD rewrite is currently local and
|
||||
pending commit/publication.
|
||||
- The accepted snapshot is
|
||||
`psd-clinical-675990d90eae51da6f2bd51b1ae2609f245772ef-snapshot`.
|
||||
- The active generation is `gen:f968b3bd7a553dbfef3cf47093698f2bc7f95f11`.
|
||||
- Retrieval acceptance reached 20/20 Hit@10.
|
||||
- A real session, `20301df7-cad7-403d-a4c1-9f35c9d07b66`, completed F1–F8 with five
|
||||
receipts, three CTEs, and a final result of 78 patients.
|
||||
- The durable acceptance record is
|
||||
`docs/testing/evidence/evidence-restructuring-psd-acceptance-2026-08-25.md`.
|
||||
|
||||
The canonical authoring, validation, publication, materialization, and preprocessing flow is
|
||||
documented in `docs/evidence.md`. The governing contracts are
|
||||
`docs/contracts/workspace-evidence-v3.md` and
|
||||
`docs/contracts/workspace-preprocessing-cli.md`.
|
||||
|
||||
## Workspace preprocessing and configuration
|
||||
|
||||
The native host CLI `tht` is the operator surface. Workspace preprocessing runs through:
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
|
||||
```
|
||||
|
||||
These commands use the profile-gated `workspace-maintenance` service. The former standalone
|
||||
preprocessing Compose fixtures are retired.
|
||||
|
||||
Workspace descriptors use schema v3. For PSD, workspace content and runtime roots point to the
|
||||
separate uncommitted repository `/Users/mp/projects/tht-workspace-psd`. Secrets remain outside
|
||||
Git and are supplied only through installation-local protected files.
|
||||
|
||||
## Database management
|
||||
|
||||
The database, table, and authoritative physical-schema catalog slices are implemented. Database
|
||||
Management now opens the Fleet Ledger presentation by default inside `AppShell`, lists every YAML
|
||||
workspace, creates at most one PostgreSQL database configuration per workspace, edits direct
|
||||
PostgreSQL, REST API, or SSH-tunnel installation bindings, replaces write-only encrypted secrets,
|
||||
and tests supported connector bindings. The surface keeps one responsive AG Grid visible at a time:
|
||||
databases lead to tables, tables lead to columns, and relationships are a sibling database view.
|
||||
Parent navigation remains explicit through the breadcrumb and emphasized back control.
|
||||
|
||||
Selection-scoped operations use one action selector plus an explicit **Run** control; ineligible
|
||||
actions remain visible with their disabled reason, while row-scoped actions stay in the pinned final
|
||||
column. The KPI strip reads installation-wide or selected-database aggregates from
|
||||
`GET /catalog/metrics`. Database configuration, metadata editors, synchronization history,
|
||||
description history, and sensitive-field review/history use the production APIs in right-side
|
||||
drawers rather than prototype fixtures; closing a history drawer does not stop its background run.
|
||||
|
||||
Physical membership, source
|
||||
comments, column types/default/nullability/PK positions, and constraint-level ordered FK pairs are
|
||||
projections of the external schema. They cannot be created, renamed, or structurally edited by
|
||||
hand, but administrators can explicitly clear catalog tables, columns, or relationships without
|
||||
touching the source database, binding, configuration, or secrets. Table deletion cascades through
|
||||
columns and relationships; table-scoped relationship cleanup includes incoming and outgoing
|
||||
relationships. Curated and generated descriptions are editable; generated descriptions start null
|
||||
and Database Management can generate or consolidate them for selected tables, selected columns,
|
||||
all targets, or only targets whose Generated Description is missing.
|
||||
|
||||
Relationship Management is now reachable directly from each configured Fleet database. One
|
||||
Relationship Map shows read-only Physical Relationships together with Generated and Manual Logical
|
||||
Relationships, with Active, Excluded, and All filters. Administrators can add a single-column
|
||||
relationship, run deterministic name/PK/type inference, exclude or restore a logical relationship,
|
||||
or delete it permanently. Exclusion retains a tombstone that a rebuild cannot reactivate; permanent
|
||||
deletion allows a later rebuild to infer the same endpoints again. Inference uses no LLM, embedding,
|
||||
or source values. It supports normalized table-qualified names, unique non-generic PK names,
|
||||
composite-PK source columns, and the `*time_key -> dim_time.<single PK>` warehouse convention while
|
||||
ignoring bare generic names. Explicit table/column metadata cleanup remains a destructive boundary: it removes
|
||||
the attached logical relationships and exclusions and requires a full schema synchronization before
|
||||
inference or runtime publication can continue.
|
||||
|
||||
The previous Database Management renderer remains a temporary comparison fallback for development
|
||||
and staging only: `?db-ui=legacy` is honored in Vite development or when
|
||||
`VITE_DB_MANAGEMENT_LEGACY=true`; it is not a production presentation. The standalone Fleet Ledger
|
||||
prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
|
||||
after which both migration aids can be removed.
|
||||
|
||||
Schema refresh is one durable asynchronous engine with database-table, selected-table-column,
|
||||
relationship, and full-database actions. Database-level menus expose only the table, relationship,
|
||||
and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections
|
||||
also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
|
||||
one-active-operation-per-database exclusion with synchronization. Runs have leases and
|
||||
restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
|
||||
apply, retained history, and a live SSE log with polling fallback. Null metadata renders blank
|
||||
rather than as a placeholder.
|
||||
|
||||
Direct PostgreSQL and strict known-host-verified OpenSSH use `pg_catalog`. REST bindings use the
|
||||
typed full-snapshot `POST /rpc/schema_snapshot` contract when available. Servers such as the
|
||||
current PSD endpoint that exposes only `POST /rpc/run_query` use one catalog-owned read-only query
|
||||
to return the exact same strict v1 snapshot in a single round trip. Both paths remain fail-closed:
|
||||
an absent capability, query error, partial result, or invalid snapshot applies no catalog changes.
|
||||
SSH is not yet enabled for NL→SQL session runtime.
|
||||
|
||||
The catalog runs in the internal `catalog-db` PostgreSQL service. Kysely migrations are an explicit
|
||||
one-shot `catalog-migrate` operation; `scripts/run-stack.sh` runs it before local startup. Runtime
|
||||
sessions now consume the Catalog's active effective relationship map through an immutable JSON
|
||||
snapshot tied to the runtime-config lease. The harness uses that snapshot as its exclusive
|
||||
relationship source while retaining Git-pinned annotations for descriptive metadata; legacy
|
||||
runtimes without a snapshot keep the previous merge behavior. The accepted design is recorded in
|
||||
`docs/plans/2026-08-26-metadata-catalog-from-thothai.md`, the snapshot contract under
|
||||
`docs/contracts/`, and ADRs 0001–0012.
|
||||
|
||||
Semantic aliases, value descriptions, synonyms, and concepts remain deferred to their dedicated
|
||||
slices.
|
||||
|
||||
AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to
|
||||
`false`, including for newly synchronized columns. An administrator may request an AI proposal based
|
||||
only on structural metadata for one selected database, selected tables, or selected columns. The
|
||||
backend divides large scopes into deterministic model requests of at most ten columns, also bounded
|
||||
by helper message size, and combines their results, but the proposal remains an unsaved draft until
|
||||
the human reviews and saves it.
|
||||
Each started suggestion attempt records a separate Sensitive Data Suggestion Run with aggregate
|
||||
counters and safe ordered events. This operational history never stores per-column proposals,
|
||||
prompts, raw model output, or provider diagnostics; reloading still discards an unsaved review
|
||||
draft.
|
||||
For unprotected columns, up to five source rows and five representative non-null values may be sent
|
||||
transiently to the configured model provider. Protected columns are omitted from source reads and
|
||||
replaced in the prompt by deterministic plausible values derived only from their metadata. Existing
|
||||
descriptions are not regenerated when a flag changes.
|
||||
|
||||
The accepted AI-description design is recorded in
|
||||
`docs/plans/2026-08-28-ai-catalog-description-generation.md`, with the formal specification in the
|
||||
adjacent `-spec.md` document and Gitea issue #4. Gitea issues #5–#11 deliver the implementation.
|
||||
The runtime deliberately keeps ThothAI's simple operating model: one installation-wide sequential
|
||||
run owned by the backend, one short-lived Python/LiteLLM completion helper per request, and
|
||||
persistence limited to the run, its safe ordered text events, and each Generated Description as
|
||||
soon as it succeeds. The helper performs at most one provider retry and never falls back to another
|
||||
model. Stop terminates the current helper and retains prior results; three consecutive exhausted
|
||||
technical batches fail the run. Startup marks stale queued/running work interrupted, and Unlock is
|
||||
available only when no local start, worker, or helper is live. Runs remain inspectable through a
|
||||
live SSE log with ordered polling fallback; there is no automatic resume or user-facing generation
|
||||
CLI. ADRs 0009–0010 record the runtime and source-sampling decisions.
|
||||
|
||||
Metadata-generation setup accepts the protected `DEEPSEEK_API_KEY` and `ZAI_API_KEY` references.
|
||||
It also accepts a model with no secret reference only when its OpenAI-compatible endpoint is
|
||||
explicit; this covers the VPN-only AritmoLab Qwen 3.6 server without creating a fake operator
|
||||
credential. The Python client supplies only its fixed non-secret compatibility placeholder.
|
||||
The AritmoLab entry also sets `disableThinking: true`, mapped to the endpoint's chat-template flag,
|
||||
because its default reasoning prose would violate the worker's exact JSON response contract.
|
||||
|
||||
Logical relationship integration with core schema-linking is complete: session creation and resume
|
||||
materialize the active physical/generated/manual map, retrieval-pack generation and Pi receive the
|
||||
same runtime config, and snapshot validation fails closed on a declared missing, invalid, or orphaned
|
||||
endpoint. Broader publication of other Catalog metadata to schema-linking remains a separate future
|
||||
slice.
|
||||
|
||||
**Deferred follow-up — Sensitive Data Policy in schema-linking.** The policy is first delivered
|
||||
and tested in catalog description generation. Its enforcement for core schema-linking remains
|
||||
out of scope until the current tickets are closed and the owner has completed the acceptance test.
|
||||
At that gate, resume the design: `tht` must receive a read-only projection of the current Sensitive
|
||||
Data Flags and exclude values from columns marked sensitive from every LSH result before it is
|
||||
given to Pi. Do not start this integration before the owner gives final approval after that test.
|
||||
|
||||
## Active deployment work and manual gates
|
||||
|
||||
### PSD server deployment program
|
||||
|
||||
The approved design and executable entry point are:
|
||||
|
||||
- `docs/plans/2026-08-20-psd-server-deployment-program-design.md`
|
||||
- `docs/plans/2026-08-20-psd-server-deployment-program.md`
|
||||
- `docs/plans/2026-08-20-psd-server-survey.md`
|
||||
- `docs/plans/2026-08-20-psd-server-project-a-standalone.md`
|
||||
- `docs/plans/2026-08-20-psd-server-project-b-authentik.md`
|
||||
|
||||
Last recorded state:
|
||||
|
||||
- survey: `SURVEY_NO_GO`;
|
||||
- Project A: `BLOCKED_BY_SURVEY_AND_MUTATION_GATE`;
|
||||
- Project B: `BLOCKED_BY_PROJECT_A_AND_PRE_B_GATE`.
|
||||
|
||||
The deployment is a clean replacement: legacy sessions, indexes, and application configuration
|
||||
are not migration inputs. The existing stack remains intact until its documented mutation and
|
||||
rollback gates are explicitly approved. Shared Omics/LocalLLM networks, ETL Evidence, DWH,
|
||||
`dwh-auth`, Supabase, Authentik, Superset, and Aritmolab are outside cleanup scope.
|
||||
|
||||
Human acceptance guides and sanitized report templates live under `docs/testing/` and
|
||||
`docs/testing/evidence/`. The remediation checklist is
|
||||
`docs/operations/psd-server-survey-remediation-checklist.md`.
|
||||
|
||||
### Authentication
|
||||
|
||||
The local/OIDC authentication remediation passed its automated review on 2026-08-18. Release and
|
||||
PSD mutation gates remain governed by:
|
||||
|
||||
- `docs/architecture/authentication.md`;
|
||||
- `docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md`;
|
||||
- `docs/operations/psd-dwh-auth-rollout.md`;
|
||||
- `docs/testing/authentication-manual-acceptance.md`.
|
||||
|
||||
Do not infer authorization for server, Nginx, Authentik, database, credential, or cutover changes
|
||||
from an automated PASS.
|
||||
|
||||
## Verification status
|
||||
|
||||
- The P1.1 workspace-directory registry and P2–P6 preprocessing workstreams are implemented and
|
||||
have automated coverage.
|
||||
- Evidence restructuring has a real PSD acceptance PASS as recorded above.
|
||||
- AI Description Generation has automated coverage across installation setup, model selection,
|
||||
generation/consolidation scopes, bounded sampling, cancellation/recovery, history, SSE/polling,
|
||||
and the LiteLLM helper boundary.
|
||||
- L2 tests requiring real providers or remote databases remain opt-in.
|
||||
- Server deployment, release, and owner-operated acceptance steps remain pending wherever the
|
||||
referenced runbooks require explicit approval.
|
||||
|
||||
Run the layer-specific checks documented in `AGENTS.md`. For release-sensitive changes, also run
|
||||
the repository contract scripts in `scripts/` and build the MkDocs site.
|
||||
|
||||
## Operational invariants
|
||||
|
||||
- `tht`'s `-c`/`--config` option follows the subcommand; it is not a global option.
|
||||
- `--json` commands write pristine JSON to stdout.
|
||||
- Persisted phase documents and the decision ledger are the source of session truth; chat is not.
|
||||
- UI chrome is English; workspace document content retains the workspace language.
|
||||
- The backend refuses resume for finalized or archived sessions.
|
||||
- A resume must send `/riprendi-sessione <id>`; a new session must send `/nuova-domanda`.
|
||||
- DWH access is read-only.
|
||||
@@ -1,437 +0,0 @@
|
||||
# ThothII
|
||||
|
||||
ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fastify/Pi/`tht`
|
||||
core. The portable deployment runs two application services plus the installation-local metadata
|
||||
catalog; DWH and LLM services remain external. Semantic services are bundled in Compose.
|
||||
|
||||
Authentication is configured through the single host CLI tht: see the [local authentication guide](docs/install/authentication-local.md),
|
||||
[generic OIDC guide](docs/install/authentication-oidc.md), and [manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
|
||||
|
||||
## Docker Compose: local startup
|
||||
|
||||
Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`,
|
||||
`catalog-db`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external,
|
||||
configurable endpoints—even when they are co-located with ThothII.
|
||||
|
||||
From a fresh clone, run these commands from the repository root:
|
||||
|
||||
```sh
|
||||
cp deploy/env/local.env.example deploy/env/local.env
|
||||
# Copy docs/install/examples/thothii-installation.local.yaml to a protected operator path,
|
||||
# replace its placeholders, chmod it 600, and set that exact THT_INSTALLATION_CONFIG_SOURCE.
|
||||
# Edit deploy/env/local.env, including PI_AUTH_FILE, THT_SECRETS_FILE, and external endpoints.
|
||||
./scripts/run-stack.sh
|
||||
```
|
||||
|
||||
The launcher builds the core, starts `catalog-db`, runs the explicit one-shot Kysely migrations,
|
||||
then runs the base+local stack in the foreground. Migrations never run implicitly in backend
|
||||
startup. The core image contains its Pi runtime; no host `pi` executable is used. For a server
|
||||
installation, build the image, start the catalog, and run the same migration service before the
|
||||
application rollout:
|
||||
|
||||
```sh
|
||||
cp deploy/env/server.env.example deploy/env/server.env
|
||||
# Prepare a mode-600 thothii-installation.yaml from the server example and set its exact
|
||||
# path as THT_INSTALLATION_CONFIG_SOURCE. Edit all remaining storage/secret/endpoint paths.
|
||||
sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example build core
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example up -d catalog-db
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example run --rm catalog-migrate
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example up --build -d
|
||||
```
|
||||
|
||||
The initializer is required for an empty or restored server Pi-state bind. It atomically creates
|
||||
the three regular targets hidden below the writable parent bind; protected Pi auth and tracked
|
||||
model/settings sources remain separate read-only mounts. See the server manual before substituting
|
||||
a root other than `/srv/thothii/pi-state`.
|
||||
|
||||
Workspace descriptors come from the Git remote configured by `THT_WORKSPACE_GIT_REMOTE`; their
|
||||
runtime endpoint and secret bindings remain installation-local. Open
|
||||
<http://127.0.0.1:8080> (set `THOTH_HTTP_PORT` in `deploy/env/local.env` to choose another
|
||||
loopback port).
|
||||
|
||||
Credentials and certificates are local protected files. Do not put them in environment examples,
|
||||
workspace YAML, URLs, or Compose interpolation values.
|
||||
|
||||
Application state is split across the named `settings`, `pi-state`, `workspace-registry`,
|
||||
`sessions`, `qdrant-data`, and `embedding-models` volumes. `docker compose down` keeps them.
|
||||
`qdrant-data` is a derived but persistent index store; `embedding-models` is an Ollama model
|
||||
cache for `qwen3-embedding:0.6b` with fixed `1024`-dimension embeddings. Only an explicit destructive command such as `docker compose
|
||||
down --volumes` removes them.
|
||||
|
||||
The frontend depends on the core health check and proxies `/health` and `/api/*` to it. The
|
||||
application health endpoint intentionally checks process readiness only; external dependency
|
||||
diagnostics are exposed by `tht doctor` and do not prevent the UI from starting.
|
||||
|
||||
## Git-backed workspace repository
|
||||
|
||||
Workspace descriptors are shared through a validated Git repository while endpoint bindings and
|
||||
secret files remain installation-local. The supported operating sequence is documented in
|
||||
[Workspace operations](docs/operations/workspaces.md); it covers curator publication, installation
|
||||
activation, runtime bindings, and preprocessing. The host setup and lifecycle path is in
|
||||
[Install and first start](docs/install/first-start.md).
|
||||
|
||||
The curator-owned repository layout is:
|
||||
|
||||
```text
|
||||
thoth-workspaces.yaml
|
||||
<id>/workspace.yaml
|
||||
<id>/evidence/**
|
||||
```
|
||||
|
||||
`thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered `workspaces` list of
|
||||
`{id, name, description?}` entries. It is authoritative for workspace ID, name, description, and
|
||||
display order. Every catalog entry must have a matching descriptor in the same commit; otherwise
|
||||
the complete candidate is rejected. Descriptors remain curator-owned and change only through a
|
||||
Git commit and push from a separate authoring clone, followed by an installation pull. ThothII
|
||||
never writes any workspace repository content.
|
||||
|
||||
The operator workflow is: curate catalog/descriptor/Evidence changes in Git, commit and push,
|
||||
**Update workspace repository** from each ThothII installation, select the workspace, complete its
|
||||
write-only runtime-secret fields, run **Validate workspace source** and **Test workspace
|
||||
connections**, then select the workspace locally before creating sessions. Each new session
|
||||
pins the Git revision it used; a later pull cannot change a Resume. Snapshot cleanup retains every
|
||||
revision referenced by an open, closed, or failed unarchived session. It reconciles from the
|
||||
single local installation list or from a server administrator's complete session list, never from
|
||||
a remote user's partial list. The isolated deployment exercise is
|
||||
`./scripts/workspace-registry-smoke.sh`; both manuals are checked with
|
||||
`./scripts/verify-workspace-install-docs.sh --profile local` or `--profile server`.
|
||||
|
||||
<!-- workspace-descriptor-contract:start -->
|
||||
Schema v3 is the only accepted workspace descriptor. Schema v1 and v2 workspace descriptors are
|
||||
rejected before activation. Candidate snapshot validation therefore makes activation or a pull fail
|
||||
atomically while the prior valid snapshot remains active. There is no in-product migrator or
|
||||
automatic conversion. A repository must already contain reviewed v3 descriptors. One workspace
|
||||
owns one Qdrant collection;
|
||||
schema, Evidence, and Memory records share that collection and stay separated by indexed payload
|
||||
`kind`.
|
||||
<!-- workspace-descriptor-contract:end -->
|
||||
|
||||
For NL→SQL runtime sessions, connector `ssh_tunnel` bindings remain diagnostic-only: their bounded
|
||||
probe cleans up the loopback forward and returns `workspace_not_activatable`; session creation is
|
||||
rejected before persistence. Database management is a separate boundary and supports a strict
|
||||
OpenSSH tunnel for **Test connection** and **Sync tables**, using a private key, optional passphrase,
|
||||
mandatory `known_hosts`, and optional PostgreSQL TLS CA/server name. Git registry access over SSH is
|
||||
unaffected. Use direct or REST connector transport for runtime sessions.
|
||||
|
||||
`docker-compose.dev.yml` is deliberately local: both published ports bind to `127.0.0.1`,
|
||||
`THT_SESSION_STORAGE=local`, and `THT_HOME=/data/local-home`. Do not set
|
||||
`THOTH_PUBLIC_EXPOSURE=true` for that profile; the backend rejects that public/local combination
|
||||
at startup.
|
||||
|
||||
Run the end-to-end packaging check with:
|
||||
|
||||
```sh
|
||||
./scripts/docker-smoke.sh
|
||||
```
|
||||
|
||||
The smoke script validates Compose, builds and waits for both services, checks health through
|
||||
the frontend, verifies SSE response headers, restarts the core, and confirms `/data` survives.
|
||||
Each run uses a unique Compose project and removes that project's containers, network, and test
|
||||
volume afterward. It never targets the fixed `thothii` operator project or its volume. Set
|
||||
`SMOKE_PROJECT` to a different explicit project name for reproducible debugging, and set
|
||||
`KEEP_SMOKE_RESOURCES=1` to retain that smoke project's resources for inspection; remove them
|
||||
later with `docker compose --project-name "$SMOKE_PROJECT" down --volumes`.
|
||||
|
||||
## Unified deployment release gates
|
||||
|
||||
Task 13 adds no-secret release gates around the canonical local and server Compose profiles. Its
|
||||
deterministic safety check does not contact the Docker daemon:
|
||||
|
||||
```sh
|
||||
bash scripts/unified-deployment-smoke.sh --self-test
|
||||
```
|
||||
|
||||
The three Linux Docker smokes are separate release commands. Each creates a unique Compose project, temporary
|
||||
Git workspace remote, fixture provider, image names, and run label. Its exit trap removes only
|
||||
resources carrying that exact run identity and never performs a global Docker prune. Cleanup
|
||||
enumerates running and stopped project containers immediately before `compose down` and refuses
|
||||
the teardown if any container, volume, or network has a foreign run label.
|
||||
|
||||
```sh
|
||||
bash scripts/unified-deployment-smoke.sh
|
||||
bash scripts/tht-update-smoke.sh
|
||||
bash scripts/server-deployment-smoke.sh
|
||||
```
|
||||
|
||||
The unified smoke builds and starts `frontend` and `core`, verifies the embedded Pi and internal
|
||||
registry, recreates with the Git remote offline, activates a valid Git update, rejects invalid Git
|
||||
content while retaining the valid snapshot, and checks the four persistence volumes. The unified
|
||||
and update-only smokes
|
||||
inject a digest-pinned non-core candidate under a deliberately mismatched Pi version and require
|
||||
`tht pi update` to roll back while preserving settings, sessions, Pi state, registry revision,
|
||||
and mount identity. The rollback candidate is the digest-pinned `hello-world` executable: a
|
||||
preflight proves that it exits successfully, so the failed replacement core satisfies
|
||||
`tht`'s stopped-core compensation precondition. The server smoke uses the same smoke-built
|
||||
core/frontend images with the server and required session overlays, disposable bind roots and
|
||||
secret files, upstream-auth checks, and a fail-closed `503` assertion for its deliberately
|
||||
unavailable disposable session endpoint. No real provider, database credential, or repository
|
||||
secret is required.
|
||||
|
||||
For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular
|
||||
`agent/auth.json`, `agent/models.json`, and `agent/settings.json` mount targets atomically before
|
||||
Compose. The server smoke starts from an empty Pi-state root and applies this same preflight; the
|
||||
real protected/tracked sources remain separate read-only mounts. Deterministic fixture tests render
|
||||
both profiles, verify that bindings stay on `core`, check mount readability, and run the production
|
||||
workspace resolver. Wrong-service, wrong-value, and broken-secret-mount mutations must fail.
|
||||
|
||||
Each public smoke has its own 30-minute process-group supervisor with TERM/KILL cleanup; CI retains
|
||||
an independent 32-minute outer timeout and does not retry a failed command.
|
||||
|
||||
Current release status (2026-08-05): clean-root render/setup and the production runtime-binding
|
||||
resolver contracts are green. The server fixture supplies all four private trusted claims,
|
||||
including exact non-admin value `0`, and a focused test proves nginx normalization produces the
|
||||
accepted non-admin backend principal. Canonical schema-v3 registry descriptors now pass through
|
||||
one backend-owned, secret-safe runtime handoff for inventory and session execution; canonical
|
||||
identity and durable session/artifact/index roots are retained. The fresh update-only smoke passed
|
||||
bad-candidate mutation, automatic `rolled_back` compensation, exact prior-image restoration,
|
||||
unchanged registry/mount identity, all four sentinels, post-rollback doctor/workspace checks, and
|
||||
exact cleanup. The one authorized server-smoke invocation was denied access to the Docker socket
|
||||
by its execution sandbox before startup, so the complete authenticated workspace and fail-closed
|
||||
session assertions still require a fresh authorized release run. Native Windows Docker
|
||||
Desktop/WSL2 remains a separate manual/self-hosted gate.
|
||||
|
||||
The deterministic native Windows contract is:
|
||||
|
||||
```powershell
|
||||
.\scripts\test-windows-clone-contract.ps1
|
||||
```
|
||||
|
||||
It checks Git's CRLF/LF attributes and bytes, copies tracked source into a temporary path containing
|
||||
spaces, builds and invokes native Windows `tht` there, and renders exactly `core` plus
|
||||
`frontend` without starting containers. On a supported self-hosted Windows Docker Desktop/WSL2
|
||||
runner, dispatch the deployment workflow with `windows_docker_startup=true`; that job executes:
|
||||
|
||||
```powershell
|
||||
.\scripts\test-windows-clone-contract.ps1 -DockerStartup
|
||||
```
|
||||
|
||||
Startup mode adds bounded image build/two-service health startup, installation-aware `tht`
|
||||
status, stopped-container-aware ownership checks, and exact cleanup. The ordinary hosted Windows
|
||||
job remains deterministic and does not claim Docker startup.
|
||||
|
||||
## Workspace preprocessing and S3 Evidence
|
||||
|
||||
Run preprocessing through the native host CLI and the installation descriptor:
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
|
||||
```
|
||||
|
||||
The CLI starts the profile-gated `workspace-maintenance` service and enforces the workspace,
|
||||
secret, Qdrant, and embedding contracts. See [Evidence](docs/evidence.md) and the
|
||||
[workspace preprocessing CLI contract](docs/contracts/workspace-preprocessing-cli.md).
|
||||
|
||||
S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance.
|
||||
AWS endpoints are used when no custom URL is supplied. Every custom endpoint is an explicit egress
|
||||
trust-boundary opt-in and uses path-style addressing; private and HTTP endpoints require additional
|
||||
independent opt-ins. Literal non-global IPv4/IPv6 addresses are classified locally; hostnames are
|
||||
not DNS-pinned, so trusted custom-endpoint deployments must enforce their destination with network
|
||||
egress policy. Store access key, secret key, and session token as secret references in
|
||||
deployment configuration—never in Compose environment values or source URIs. Discovery and reads
|
||||
are bounded by configured page, object, and byte limits.
|
||||
|
||||
Create a versioned Qdrant volume backup for one exact Compose project (the filename is
|
||||
operator-controlled, so use an immutable timestamp or release identifier):
|
||||
|
||||
```sh
|
||||
./scripts/vector-backup.sh \
|
||||
--project-name thothii \
|
||||
--output /secure/backups/thoth-qdrant-2026-08-08.tar
|
||||
```
|
||||
|
||||
The script resolves exactly one Docker volume with the labels
|
||||
`com.docker.compose.project=<project>` and `com.docker.compose.volume=qdrant-data`, stops the
|
||||
`qdrant` service if it is running, archives that volume's persistent contents, then restores the
|
||||
prior service state. It never performs global Docker cleanup and refuses to overwrite an existing
|
||||
archive path.
|
||||
|
||||
Restore targets that same exact project-scoped `qdrant-data` volume. Because restore replaces the
|
||||
persistent Qdrant data in place, it requires an explicit confirmation that exactly repeats the
|
||||
Compose project name by passing `--confirm-project`:
|
||||
|
||||
```sh
|
||||
./scripts/vector-restore.sh \
|
||||
--project-name thothii \
|
||||
--input /secure/backups/thoth-qdrant-2026-08-08.tar \
|
||||
--confirm-project thothii
|
||||
```
|
||||
|
||||
The restore script stops `qdrant`, validates the exact labeled target, stages the current volume
|
||||
contents for rollback, extracts the requested archive into the volume, and then returns the
|
||||
service to its prior running state. It restores semantic storage only. Before reopening write
|
||||
traffic, the workspace registry must already be at a reviewed v3 descriptor revision compatible
|
||||
with the restored collection; then run backend health checks and a known retrieval query. The
|
||||
helper does not restore descriptors, rename collections, or reconcile an incompatible collection
|
||||
contract.
|
||||
|
||||
## Production trust boundary and secrets
|
||||
|
||||
ThothII does not implement OIDC. Do not expose its application port directly to a network.
|
||||
The production pattern is an authenticated host reverse proxy that:
|
||||
|
||||
- terminates TLS and authenticates every request;
|
||||
- removes any client-supplied identity header;
|
||||
- injects one trusted `X-Authenticated-User` value;
|
||||
- proxies to the loopback-only ThothII frontend.
|
||||
|
||||
[`deploy/nginx-authenticated-proxy.conf.example`](deploy/nginx-authenticated-proxy.conf.example)
|
||||
shows the contract using nginx `auth_request`; replace the placeholder authentication gateway
|
||||
with the organization's reviewed identity proxy. `AUTH_MODE=upstream` trusts this boundary and
|
||||
rejects requests without the identity header. Setting `THOTH_PUBLIC_EXPOSURE=true` with any other
|
||||
auth mode fails during core startup.
|
||||
|
||||
Production credentials use the existing Compose secret-bundle contract, never environment values.
|
||||
Copy `deploy/secrets/thothii.secrets.example` to a protected host file, include only the required
|
||||
keys, and set its absolute path as `THT_SECRETS_FILE` in the operator env. Keep Pi's native
|
||||
provider auth in the separate protected file named by `PI_AUTH_FILE`.
|
||||
|
||||
Description Generation is configured independently in the protected installation descriptor under
|
||||
`metadataGeneration`. Set `THT_INSTALLATION_CONFIG_SOURCE` to that exact host file; Compose mounts
|
||||
it read-only into `core` and supplies the fixed runtime `THT_INSTALLATION_CONFIG_FILE` path. Each
|
||||
keyed model stores only an audited `apiKeyEnv` reference. The referenced value stays in the secret
|
||||
bundle; a model may omit `apiKeyEnv` only when it declares an explicit endpoint that accepts
|
||||
unauthenticated requests. The browser receives only model IDs, labels, and the configured default.
|
||||
Configuration changes take effect after restart and do not use Pi settings or workspace
|
||||
`llm_policy`.
|
||||
|
||||
Before enabling Description Generation, approve the selected model provider for bounded source-data
|
||||
disclosure. Every catalog column has a **Sensitive** flag that defaults to `false`. Administrators can
|
||||
request an AI proposal based only on structural metadata, then must review and save the resulting
|
||||
checkboxes themselves. The proposal never reads column contents and is not persisted automatically.
|
||||
|
||||
For unprotected columns, a request may send up to five real source rows and five representative
|
||||
distinct, non-null example values. Protected columns are omitted from source reads and replaced in the
|
||||
prompt by deterministic plausible values derived only from column metadata. Samples are transient and
|
||||
are not stored in generation runs, run logs, application logs, API responses, or catalog metadata;
|
||||
prompt and sample snapshots are not retained. A flag change applies to later generations and does not
|
||||
regenerate existing descriptions.
|
||||
|
||||
Description Generation is an interactive Database Management operation, not a user-facing CLI.
|
||||
The installation runs at most one sequential generation at a time. The run drawer exposes safe
|
||||
ordered events through SSE with polling fallback, Stop terminates the current helper while keeping
|
||||
already stored results, and Run history retains terminal runs for inspection. A backend restart
|
||||
marks queued or running work interrupted instead of resuming it; use Generate Missing to continue.
|
||||
Unlock is reserved for a stale recorded run and is rejected while a local start, worker, or helper
|
||||
is still live.
|
||||
|
||||
The bundle is mounted read-only as `/run/secrets/thothii.secrets` and must be mode `0600` or
|
||||
`0400` on the host. Docker's runtime `0444` mode is accepted only beneath `/run/secrets`; see
|
||||
[`deploy/secrets/README.md`](deploy/secrets/README.md). A PEM CA chain is deliberately not a
|
||||
bundle value: PEM contains whitespace and is rejected by the strict parser. Keep the CA chain in
|
||||
the host/secret-manager materialization and add a reviewed Compose override that mounts it at
|
||||
`/run/secrets/ca-chain.pem` and sets `THT_SSL_CA` when a private CA is required. The base bundle
|
||||
does not create that mount. The frontend remains on loopback; the authenticated host proxy is the
|
||||
only public listener.
|
||||
|
||||
Set the selected model provider in application settings (or `PI_PROVIDER`). For each Pi spawn the
|
||||
backend validates and reads `THT_MODEL_API_KEY` from the bundle, then exposes its value only as the provider's
|
||||
recognized child variable (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or
|
||||
`ZAI_API_KEY`). Neither the generic file path nor deprecated `PI_PROVIDER_API_KEY` is inherited by
|
||||
Pi. Local providers such as Ollama require no model key.
|
||||
|
||||
`THT_MODEL_API_KEY` supports Pi providers whose authentication is exactly one key:
|
||||
`ant-ling`, `anthropic`, `cerebras`, `deepseek`, `fireworks`, `github-copilot`, `google`
|
||||
(including the `gemini` alias), `google-vertex` when using its API-key mode, `groq`,
|
||||
`huggingface`, `kimi-coding`, `minimax`, `minimax-cn`, `mistral`, `moonshotai`,
|
||||
`moonshotai-cn`, `nvidia`, `openai`, `opencode`, `opencode-go`, `openrouter`, `together`,
|
||||
`vercel-ai-gateway`, `xai`, the four `xiaomi*` providers, `zai`, and `zai-coding-cn`.
|
||||
Compound providers are deliberately unsupported: `amazon-bedrock`, `azure-openai-responses`,
|
||||
`cloudflare-workers-ai`, and `cloudflare-ai-gateway` require multiple credential/configuration
|
||||
values. Selecting one fails before Pi starts; ambient AWS, Azure, and Cloudflare credentials are
|
||||
still scrubbed. Supporting them requires a future dedicated provider-specific configuration.
|
||||
|
||||
## User-owned session server cutover
|
||||
|
||||
The server profile stores sessions and per-user preferences directly in PostgreSQL schema
|
||||
`thoth_sessions`; it does not use PostgREST, browser storage, a shared session directory, or a
|
||||
dual write. Use [`deploy/compose.session-server.yaml.example`](deploy/compose.session-server.yaml.example)
|
||||
with the canonical base+server files and set `THT_SERVER_WORKSPACE_CONFIG` to an absolute,
|
||||
protected copy of [`deploy/workspaces/server-sessions.yaml.example`](deploy/workspaces/server-sessions.yaml.example).
|
||||
|
||||
The runtime login needs membership in the no-login database role `thoth_sessions_runtime` only.
|
||||
The distinct, one-shot migrator login needs migration authority and uses
|
||||
`thoth_sessions_migrator`; it must never be mounted into `core`. Set the non-secret endpoint and
|
||||
role fields in the protected deployment environment:
|
||||
|
||||
```dotenv
|
||||
AUTH_MODE=upstream
|
||||
THOTH_PUBLIC_EXPOSURE=true
|
||||
THT_SESSION_STORAGE=postgres
|
||||
THT_SESSION_DB_HOST=sessions-db.internal
|
||||
THT_SESSION_DB_PORT=5432
|
||||
THT_SESSION_DB_NAME=thoth
|
||||
THT_SESSION_RUNTIME_USER=thoth_sessions_app
|
||||
THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate
|
||||
THT_SESSION_DB_SSLMODE=verify-full
|
||||
THT_SESSION_RUNTIME_PASSWORD_SOURCE=/secure/thoth/session-runtime-password
|
||||
THT_SESSION_MIGRATOR_PASSWORD_SOURCE=/secure/thoth/session-migrator-password
|
||||
THT_SESSION_CA_SOURCE=/secure/thoth/session-ca.pem
|
||||
```
|
||||
|
||||
The overlay mounts the runtime password at `/run/secrets/session_runtime_password`, the CA at
|
||||
`/run/secrets/session_ca.pem`, and passes those paths—not their contents—to the server workspace.
|
||||
It mounts `session_migrator_password` only to `session-migrate`. The backend refuses a server
|
||||
session store without upstream authentication, direct DB host/name/runtime user/password-file,
|
||||
`verify-ca` or `verify-full`, and an absolute CA path.
|
||||
The migrator independently rejects every other TLS mode before reading its password secret or
|
||||
constructing a database URL.
|
||||
|
||||
Perform the cutover in one maintenance window, with the upstream identity-proxy headers and
|
||||
backend principal parser deployed together. Neither change is safe to deploy independently: the
|
||||
proxy clears the legacy identity header and the backend rejects it. Drain/stop active Pi work,
|
||||
enable a maintenance response at the proxy, then run the migrator once and inspect its pristine JSON:
|
||||
|
||||
```sh
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml -f deploy/compose.session-server.yaml.example \
|
||||
--profile session-migrate run --rm session-migrate
|
||||
```
|
||||
|
||||
It must report no pending or drifted migrations before starting the replacement core. `/health`
|
||||
is a liveness probe and remains `200`; any request that needs unavailable repository storage
|
||||
returns a fixed `503` before a Pi process starts. Verify this with an authenticated request after
|
||||
the replacement core is healthy, then remove maintenance mode.
|
||||
|
||||
Do not import the three legacy server filesystem sessions: they have no trusted owner binding.
|
||||
During the same maintenance window, archive the exact three reviewed IDs, verify the generated
|
||||
archive and `.sha256`, then rerun the command with `--delete` to remove only those three source
|
||||
directories:
|
||||
|
||||
```sh
|
||||
./docker/cutover-legacy-sessions.sh \
|
||||
/secure/thoth/legacy-sessions /secure/backups/thoth-legacy-sessions-2026-07-16.tar \
|
||||
SESSION_ID_1 SESSION_ID_2 SESSION_ID_3
|
||||
# After independent archive review, use a new backup filename:
|
||||
./docker/cutover-legacy-sessions.sh --delete \
|
||||
/secure/thoth/legacy-sessions /secure/backups/thoth-legacy-sessions-2026-07-16-delete.tar \
|
||||
SESSION_ID_1 SESSION_ID_2 SESSION_ID_3
|
||||
```
|
||||
|
||||
The helper refuses to overwrite an existing backup and refuses any count other than three
|
||||
distinct IDs. Never run it against a live path without the maintenance gate. Roll back application
|
||||
code only by keeping PostgreSQL as the single source of truth and deploying a compatible fixed
|
||||
release. Do not restore filesystem persistence, do not re-import the archive, and never dual-write
|
||||
sessions to database and files.
|
||||
|
||||
## Reproducible image verification
|
||||
|
||||
Base images use exact tags and immutable multi-platform manifest digests. Dependency update and
|
||||
residual OS-repository limitations are documented in [`docker/LOCKS.md`](docker/LOCKS.md).
|
||||
Run the shared architecture gate with `PLATFORM=linux/amd64` or `PLATFORM=linux/arm64`:
|
||||
|
||||
```sh
|
||||
PLATFORM=linux/arm64 ./scripts/verify-container-images.sh
|
||||
```
|
||||
|
||||
It builds both images, runs common version/runtime/security smokes, and emits an image/package
|
||||
inventory beneath `.artifacts/container-images/`. CI runs the same script for both architectures.
|
||||
@@ -1 +0,0 @@
|
||||
data/settings.json
|
||||
Generated
-4974
File diff suppressed because it is too large
Load Diff
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"name": "thothii-backend",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"prebuild": "node scripts/clean-dist.mjs",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"catalog:migrate": "node dist/catalog/migrate.js",
|
||||
"test": "vitest run",
|
||||
"start": "node dist/server.js",
|
||||
"test:schema-v3-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fastify/cookie": "11.1.2",
|
||||
"@fastify/cors": "^11.2.0",
|
||||
"@fastify/rate-limit": "11.2.0",
|
||||
"@types/pg": "^8.20.3",
|
||||
"fastify": "^5.0.0",
|
||||
"kysely": "^0.29.5",
|
||||
"openid-client": "6.8.5",
|
||||
"pg": "^8.22.0",
|
||||
"yaml": "^2.9.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testcontainers/postgresql": "^12.1.0",
|
||||
"@types/node": "24.13.3",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0",
|
||||
"vitest": "^2.1.0"
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
/** Shared Bash heredoc word parser for descriptor extraction and policy masking. */
|
||||
|
||||
function physicalLines(source) {
|
||||
const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? [];
|
||||
if (rawLines.length === 0) rawLines.push("");
|
||||
let offset = 0;
|
||||
return rawLines.map((raw) => {
|
||||
const record = { raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, ""), start: offset };
|
||||
offset += raw.length;
|
||||
return record;
|
||||
});
|
||||
}
|
||||
|
||||
function heredocOperator(line) {
|
||||
let quote = null;
|
||||
let arithmeticDepth = 0;
|
||||
for (let index = 0; index < line.length - 1; index += 1) {
|
||||
const character = line[index];
|
||||
if (quote !== null) {
|
||||
if (character === quote) quote = null;
|
||||
else if (quote === '"' && character === "\\") index += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "'" || character === '"') { quote = character; continue; }
|
||||
if (character === "\\") { index += 1; continue; }
|
||||
if (character === "#" && (index === 0 || /[ \t;|&()]/u.test(line[index - 1]))) break;
|
||||
if (character === "(" && line[index + 1] === "(") { arithmeticDepth += 1; index += 1; continue; }
|
||||
if (character === ")" && line[index + 1] === ")" && arithmeticDepth > 0) { arithmeticDepth -= 1; index += 1; continue; }
|
||||
if (arithmeticDepth > 0 || character !== "<" || line[index + 1] !== "<") continue;
|
||||
if (line[index - 1] === "<" || line[index + 2] === "<") { index += 1; continue; }
|
||||
return index;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function endsWithBashContinuation(line) {
|
||||
let quote = null;
|
||||
for (let index = 0; index < line.length; index += 1) {
|
||||
const character = line[index];
|
||||
if (quote === null && character === "`") { index += 1; continue; }
|
||||
if (quote === "'") { if (character === "'") quote = null; continue; }
|
||||
if (character === '"') { if (quote === '"') quote = null; else if (quote === null) quote = '"'; continue; }
|
||||
if (character !== "\\") continue;
|
||||
if (index === line.length - 1) return true;
|
||||
if (quote === null || (quote === '"' && '$`"\\'.includes(line[index + 1]))) index += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function bashLogicalLine(lines, start) {
|
||||
let line = lines[start];
|
||||
let end = start;
|
||||
while (endsWithBashContinuation(line)) {
|
||||
if (end + 1 >= lines.length) break;
|
||||
line = `${line.slice(0, -1)}${lines[end + 1]}`;
|
||||
end += 1;
|
||||
}
|
||||
return { line, end };
|
||||
}
|
||||
|
||||
function bashHeredocOpener(line, operator, label, lineNumber) {
|
||||
let cursor = operator + 2;
|
||||
let stripTabs = false;
|
||||
if (line[cursor] === "-") { stripTabs = true; cursor += 1; }
|
||||
while (line[cursor] === " " || line[cursor] === "\t") cursor += 1;
|
||||
const unsupported = () => { throw new Error(`${label}:${lineNumber}: unsupported Bash heredoc opener`); };
|
||||
if (cursor >= line.length || line[cursor] === "#") unsupported();
|
||||
let delimiter = "";
|
||||
let quotedDelimiter = false;
|
||||
while (cursor < line.length) {
|
||||
const character = line[cursor];
|
||||
if (character === " " || character === "\t" || ";|&<>".includes(character)) break;
|
||||
if (character === "'" || character === '"') {
|
||||
quotedDelimiter = true;
|
||||
const quote = character;
|
||||
cursor += 1;
|
||||
let closed = false;
|
||||
while (cursor < line.length) {
|
||||
const quoted = line[cursor];
|
||||
if (quoted === quote) { closed = true; cursor += 1; break; }
|
||||
if (quote === '"' && quoted === "\\") {
|
||||
cursor += 1;
|
||||
if (cursor >= line.length) unsupported();
|
||||
const escaped = line[cursor];
|
||||
delimiter += '$`"\\'.includes(escaped) ? escaped : `\\${escaped}`;
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
delimiter += quoted;
|
||||
cursor += 1;
|
||||
}
|
||||
if (!closed) unsupported();
|
||||
continue;
|
||||
}
|
||||
if (character === "\\") {
|
||||
quotedDelimiter = true;
|
||||
cursor += 1;
|
||||
if (cursor >= line.length) unsupported();
|
||||
delimiter += line[cursor];
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "$" || character === "`" || "(){}[]*?".includes(character)) unsupported();
|
||||
delimiter += character;
|
||||
cursor += 1;
|
||||
}
|
||||
if (delimiter.length === 0) unsupported();
|
||||
if (heredocOperator(line.slice(cursor)) >= 0) unsupported();
|
||||
return { delimiter, stripTabs, expandable: !quotedDelimiter };
|
||||
}
|
||||
|
||||
function parsedBashHeredocs(source, label) {
|
||||
const records = physicalLines(source);
|
||||
const lines = records.map((record) => record.text);
|
||||
const extracted = [];
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const logical = bashLogicalLine(lines, index);
|
||||
const operator = heredocOperator(logical.line);
|
||||
if (operator < 0) { index = logical.end; continue; }
|
||||
const opener = index;
|
||||
const { delimiter, stripTabs, expandable } = bashHeredocOpener(logical.line, operator, label, index + 1);
|
||||
index = logical.end;
|
||||
const body = [];
|
||||
const startLine = index + 2;
|
||||
const bodyStart = records[index + 1]?.start ?? source.length;
|
||||
let closed = false;
|
||||
for (index += 1; index < lines.length; index += 1) {
|
||||
const candidate = stripTabs ? lines[index].replace(/^\t+/u, "") : lines[index];
|
||||
if (candidate === delimiter) { closed = true; break; }
|
||||
body.push(candidate);
|
||||
}
|
||||
const bodyEnd = closed ? records[index].start : source.length;
|
||||
extracted.push({
|
||||
source: `${body.join("\n")}\n`, label: `${label}:${startLine} Bash heredoc${closed ? "" : " (unclosed)"}`,
|
||||
expandable, closed, bodyStart, bodyEnd, path: label,
|
||||
rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""),
|
||||
});
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
|
||||
function extractBashDocuments(source, label) {
|
||||
return parsedBashHeredocs(source, label).map(({ bodyStart: _start, bodyEnd: _end, closed: _closed, ...document }) => document);
|
||||
}
|
||||
|
||||
function literalBashHeredocBodyRanges(source, label) {
|
||||
const ranges = [];
|
||||
for (const heredoc of parsedBashHeredocs(source, label)) {
|
||||
if (!heredoc.expandable) {
|
||||
if (!heredoc.closed) throw new Error(`${label}: revision-state policy found an unclosed literal Bash heredoc`);
|
||||
ranges.push({ start: heredoc.bodyStart, end: heredoc.bodyEnd });
|
||||
}
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
export { extractBashDocuments, literalBashHeredocBodyRanges };
|
||||
@@ -1,13 +0,0 @@
|
||||
import { rm } from "node:fs/promises";
|
||||
import { basename, dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const scriptDirectory = dirname(fileURLToPath(import.meta.url));
|
||||
const backendRoot = resolve(scriptDirectory, "..");
|
||||
const target = resolve(backendRoot, "dist");
|
||||
|
||||
if (dirname(target) !== backendRoot || basename(target) !== "dist") {
|
||||
throw new Error(`Refusing to clean non-dist target: ${target}`);
|
||||
}
|
||||
|
||||
await rm(target, { recursive: true, force: true });
|
||||
@@ -1,147 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
access, cp, lstat, mkdir, mkdtemp, readFile, rm, symlink, writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const backendRoot = fileURLToPath(new URL("../", import.meta.url));
|
||||
const ownedRoots = [];
|
||||
|
||||
function packageBuildInvocation(platform = process.platform, environment = process.env) {
|
||||
if (platform === "win32") {
|
||||
const comspec = environment.ComSpec ?? environment.COMSPEC;
|
||||
if (!comspec) throw new Error("ComSpec is required to run npm on Windows.");
|
||||
return { executable: comspec, args: ["/d", "/s", "/c", "npm.cmd run build"] };
|
||||
}
|
||||
return { executable: "npm", args: ["run", "build"] };
|
||||
}
|
||||
|
||||
async function isMissing(path) {
|
||||
try {
|
||||
await access(path);
|
||||
return false;
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return true;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function createOwnedRoot(prefix) {
|
||||
const root = await mkdtemp(join(tmpdir(), prefix));
|
||||
ownedRoots.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
async function copyCleaner(fixtureRoot) {
|
||||
await mkdir(join(fixtureRoot, "scripts"), { recursive: true });
|
||||
const cleaner = join(fixtureRoot, "scripts", "clean-dist.mjs");
|
||||
await cp(join(backendRoot, "scripts", "clean-dist.mjs"), cleaner);
|
||||
return cleaner;
|
||||
}
|
||||
|
||||
async function createBackendFixture() {
|
||||
const fixtureRoot = await createOwnedRoot("thoth-backend-clean-dist-");
|
||||
await Promise.all([
|
||||
cp(join(backendRoot, "package.json"), join(fixtureRoot, "package.json")),
|
||||
cp(join(backendRoot, "tsconfig.json"), join(fixtureRoot, "tsconfig.json")),
|
||||
cp(join(backendRoot, "src"), join(fixtureRoot, "src"), { recursive: true }),
|
||||
copyCleaner(fixtureRoot),
|
||||
]);
|
||||
const dependencyRoot = join(backendRoot, "node_modules");
|
||||
const dependencyEntry = await lstat(dependencyRoot);
|
||||
if (!dependencyEntry.isDirectory() || dependencyEntry.isSymbolicLink()) {
|
||||
throw new Error("Backend node_modules must be a real directory.");
|
||||
}
|
||||
await symlink(
|
||||
dependencyRoot,
|
||||
join(fixtureRoot, "node_modules"),
|
||||
process.platform === "win32" ? "junction" : "dir",
|
||||
);
|
||||
return fixtureRoot;
|
||||
}
|
||||
|
||||
async function removeOwnedRoot(root) {
|
||||
for (const childName of ["node_modules", "dist"]) {
|
||||
const child = join(root, childName);
|
||||
try {
|
||||
const entry = await lstat(child);
|
||||
if (entry.isSymbolicLink()) {
|
||||
await rm(child, { recursive: true, force: true });
|
||||
} else if (childName === "node_modules") {
|
||||
throw new Error(`Refusing to clean fixture with a non-link node_modules: ${root}`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.code !== "ENOENT") throw error;
|
||||
}
|
||||
}
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
for (const root of ownedRoots.splice(0)) await removeOwnedRoot(root);
|
||||
});
|
||||
|
||||
test("Windows package builds use ComSpec instead of executing npm.cmd directly", () => {
|
||||
assert.deepEqual(
|
||||
packageBuildInvocation("win32", { ComSpec: "C:\\Windows\\System32\\cmd.exe" }),
|
||||
{
|
||||
executable: "C:\\Windows\\System32\\cmd.exe",
|
||||
args: ["/d", "/s", "/c", "npm.cmd run build"],
|
||||
},
|
||||
);
|
||||
assert.throws(() => packageBuildInvocation("win32", {}), /ComSpec is required/);
|
||||
});
|
||||
|
||||
test("cleaner is idempotent and removes a dist link without following it", async () => {
|
||||
const fixtureRoot = await createOwnedRoot("thoth-backend-cleaner-");
|
||||
const cleaner = await copyCleaner(fixtureRoot);
|
||||
const fixtureDist = join(fixtureRoot, "dist");
|
||||
|
||||
await execFileAsync(process.execPath, [cleaner], { cwd: fixtureRoot });
|
||||
assert.equal(await isMissing(fixtureDist), true);
|
||||
await execFileAsync(process.execPath, [cleaner], { cwd: fixtureRoot });
|
||||
assert.equal(await isMissing(fixtureDist), true);
|
||||
|
||||
const outsideRoot = await createOwnedRoot("thoth-backend-cleaner-outside-");
|
||||
const outsideSentinel = join(outsideRoot, "sentinel.txt");
|
||||
await writeFile(outsideSentinel, "outside-owned-data\n", "utf8");
|
||||
await symlink(outsideRoot, fixtureDist, process.platform === "win32" ? "junction" : "dir");
|
||||
|
||||
await execFileAsync(process.execPath, [cleaner], { cwd: fixtureRoot });
|
||||
|
||||
assert.equal(await isMissing(fixtureDist), true, "dist link survived cleaner");
|
||||
assert.equal(await readFile(outsideSentinel, "utf8"), "outside-owned-data\n");
|
||||
});
|
||||
|
||||
test("package build replaces the complete backend distribution in an owned fixture", async () => {
|
||||
const fixtureRoot = await createBackendFixture();
|
||||
const copiedPackage = JSON.parse(await readFile(join(fixtureRoot, "package.json"), "utf8"));
|
||||
assert.equal(copiedPackage.scripts.prebuild, "node scripts/clean-dist.mjs");
|
||||
|
||||
const workspacesDist = join(fixtureRoot, "dist", "workspaces");
|
||||
const staleModules = [
|
||||
"stale-build-sentinel.js",
|
||||
"migrate-legacy.js",
|
||||
"migrate-v2-qdrant.js",
|
||||
].map((name) => join(workspacesDist, name));
|
||||
await mkdir(workspacesDist, { recursive: true });
|
||||
await Promise.all(staleModules.map((path) => writeFile(path, "export const stale = true;\n", "utf8")));
|
||||
|
||||
const { executable, args } = packageBuildInvocation();
|
||||
await execFileAsync(executable, args, { cwd: fixtureRoot });
|
||||
|
||||
for (const path of staleModules) {
|
||||
assert.equal(await isMissing(path), true, `stale module survived the package build: ${path}`);
|
||||
}
|
||||
assert.equal(
|
||||
await isMissing(join(fixtureRoot, "dist", "server.js")),
|
||||
false,
|
||||
"server output was not compiled",
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,958 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
chmod, cp, lstat, mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createServer, connect } from "node:net";
|
||||
import dgram from "node:dgram";
|
||||
import { Worker } from "node:worker_threads";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
canonicalIntegrationBase,
|
||||
CHECK_IDS,
|
||||
buildSafeEnvironment,
|
||||
collectRepositoryProvenance,
|
||||
installExternalFetchGuard,
|
||||
installNetworkGuard,
|
||||
installProductionSurfaceGuard,
|
||||
resolveProductionExecutables,
|
||||
negativeRequestEvidence,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
deriveOverall,
|
||||
executeChecks,
|
||||
exportArchiveEvidencePath,
|
||||
readAndValidateOwnership,
|
||||
runCommand,
|
||||
runIntegration,
|
||||
scalarSecretBytes,
|
||||
scanSecrets,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p1-acceptance.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p1 acceptance repository with spaces-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
return await realpath(root);
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p1-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
join(dirname(base), id),
|
||||
]) assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p1-${"A".repeat(32)}`), `p1-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses every unowned or ambiguous root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const cases = [
|
||||
["missing ownership", async (run) => rm(join(run.root, "ownership.json"))],
|
||||
["malformed ownership", async (run) => writeFile(join(run.root, "ownership.json"), "{")],
|
||||
["mismatched root", async (run) => {
|
||||
const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8"));
|
||||
value.root = join(base, `p1-${"b".repeat(32)}`);
|
||||
await writeFile(join(run.root, "ownership.json"), JSON.stringify(value));
|
||||
}],
|
||||
["mismatched pid", async (run) => {
|
||||
const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8"));
|
||||
value.pid += 1;
|
||||
await writeFile(join(run.root, "ownership.json"), JSON.stringify(value));
|
||||
}],
|
||||
["wrong resource list", async (run) => {
|
||||
const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8"));
|
||||
value.resources.push(join(repositoryRoot, "foreign"));
|
||||
await writeFile(join(run.root, "ownership.json"), JSON.stringify(value));
|
||||
}],
|
||||
];
|
||||
for (const [, mutate] of cases) {
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await mutate(run);
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }));
|
||||
assert.equal((await lstat(run.root)).isDirectory(), true);
|
||||
}
|
||||
const wrongNonce = await createOwnedRun({ repositoryRoot });
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: wrongNonce.root, expectedNonce: "0".repeat(64) }));
|
||||
const symlinkRun = await createOwnedRun({ repositoryRoot });
|
||||
const target = `${symlinkRun.root}-target`;
|
||||
await rm(symlinkRun.root, { recursive: true });
|
||||
await mkdir(target);
|
||||
await symlink(target, symlinkRun.root);
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: symlinkRun.root, expectedNonce: symlinkRun.nonce }));
|
||||
for (const bad of [base, join(repositoryRoot, ".artifacts", "manual-acceptance"), join(base, "foreign")]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: "0".repeat(64) }));
|
||||
}
|
||||
});
|
||||
|
||||
test("cleanup atomically removes one owned root and preserves siblings", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p1-${"c".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(lstat(run.root));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id, status: "PASS", startedAt: "2026-08-09T00:00:00.000Z",
|
||||
finishedAt: "2026-08-09T00:00:01.000Z", commands: ["git"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
function validReport(checks = CHECK_IDS.map(resultFor)) {
|
||||
return {
|
||||
schemaVersion: 1, runId: `p1-${"d".repeat(32)}`, startedAt: "2026-08-09T00:00:00.000Z",
|
||||
finishedAt: "2026-08-09T00:00:02.000Z", command: "p1-acceptance integration --keep",
|
||||
overall: deriveOverall(checks), checks,
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation enforces uniqueness, derivation, safe evidence, hashes, times, and commands", () => {
|
||||
assert.doesNotThrow(() => validateReport(validReport()));
|
||||
const mutations = [
|
||||
(r) => r.checks.push(structuredClone(r.checks[0])),
|
||||
(r) => { r.checks[0].attempt = 1; },
|
||||
(r) => { r.checks[0].artifacts[0].path = "../secret"; },
|
||||
(r) => { r.checks[0].artifacts[0].sha256 = "bad"; },
|
||||
(r) => { r.checks[0].startedAt = "today"; },
|
||||
(r) => { r.checks[0].commands = ["git status"]; },
|
||||
(r) => { r.overall = "PASS"; r.checks[0].status = "FAIL"; },
|
||||
(r) => { r.nested = { retries: 2 }; },
|
||||
];
|
||||
for (const mutate of mutations) {
|
||||
const report = validReport(); mutate(report); assert.throws(() => validateReport(report));
|
||||
}
|
||||
});
|
||||
|
||||
function exactScenarios(run = async () => ({ commands: [], artifacts: [] })) {
|
||||
return CHECK_IDS.map((id) => ({ id, run: () => run(id) }));
|
||||
}
|
||||
|
||||
test("injected failure executes once, retains a complete ordered diagnostic report, and returns nonzero", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const calls = [];
|
||||
const failAt = CHECK_IDS[3];
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: false, failAt,
|
||||
checks: exactScenarios(async (id) => { calls.push(id); return { commands: [], artifacts: [] }; }),
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.deepEqual(calls, CHECK_IDS.slice(0, 4));
|
||||
assert.equal((await lstat(result.runRoot)).isDirectory(), true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS);
|
||||
assert.equal(report.checks.filter((check) => check.status === "FAIL").length, CHECK_IDS.length - 3);
|
||||
assert.equal(report.checks[3].error, "Acceptance scenario failed safely.");
|
||||
assert.equal(report.checks[4].error, "Not executed after earlier failure.");
|
||||
});
|
||||
|
||||
test("failed scenario retains partial request and response evidence with observed commands", async () => {
|
||||
const partial = {
|
||||
commands: ["git"],
|
||||
artifacts: [
|
||||
{ path: "requests/partial.json", sha256: "a".repeat(64) },
|
||||
{ path: "responses/partial.json", sha256: "b".repeat(64) },
|
||||
],
|
||||
};
|
||||
const checks = exactScenarios(async (id) => {
|
||||
if (id === CHECK_IDS[4]) {
|
||||
const error = new Error("HTTP scenario failed after response persistence");
|
||||
error.acceptancePartial = partial;
|
||||
throw error;
|
||||
}
|
||||
return {};
|
||||
});
|
||||
const results = await executeChecks({ checks });
|
||||
assert.deepEqual(results[4].commands, partial.commands);
|
||||
assert.deepEqual(results[4].artifacts, partial.artifacts);
|
||||
assert.equal(results[4].error, "Acceptance scenario failed safely.");
|
||||
});
|
||||
|
||||
test("executeChecks never repeats or executes after first failure but emits the exact check set", async () => {
|
||||
const calls = new Map();
|
||||
const result = await executeChecks({
|
||||
checks: exactScenarios(async (id) => { calls.set(id, (calls.get(id) ?? 0) + 1); return {}; }),
|
||||
failAt: CHECK_IDS[1],
|
||||
});
|
||||
assert.deepEqual(result.map(({ id }) => id), CHECK_IDS);
|
||||
assert.deepEqual(Object.fromEntries(calls), Object.fromEntries(CHECK_IDS.slice(0, 2).map((id) => [id, 1])));
|
||||
assert.equal(result[1].status, "FAIL");
|
||||
assert(result.slice(2).every(({ status, error }) => status === "FAIL" && error === "Not executed after earlier failure."));
|
||||
assert.throws(() => validateReport(validReport(CHECK_IDS.slice(0, -1).map(resultFor))));
|
||||
await assert.rejects(executeChecks({ checks: exactScenarios().reverse() }));
|
||||
});
|
||||
|
||||
test("owned setup failure still writes one safe result for every exact check", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: false,
|
||||
setup: async () => { throw new Error("fixture setup raw failure"); },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS);
|
||||
assert.equal(report.checks[0].error, "Acceptance setup failed safely.");
|
||||
assert(report.checks.slice(1).every(({ error }) => error === "Not executed after earlier failure."));
|
||||
});
|
||||
|
||||
test("scalar fixture secret files contain no harness-invalid whitespace", () => {
|
||||
const bytes = scalarSecretBytes("CANARY-secret-value-123456");
|
||||
assert.equal(bytes.toString("utf8"), "CANARY-secret-value-123456");
|
||||
assert.equal([...bytes].some((byte) => /\s/.test(String.fromCharCode(byte))), false);
|
||||
assert.throws(() => scalarSecretBytes("bad secret"));
|
||||
});
|
||||
|
||||
test("secret scanner excludes only the direct fixture-secrets subtree", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const canary = "CANARY-secret-value-123456";
|
||||
await mkdir(join(run.root, "fixture-secrets"));
|
||||
await writeFile(join(run.root, "fixture-secrets", "allowed"), canary);
|
||||
const paths = [
|
||||
"logs/a.log", "responses/a.json", "rendered/a.yaml", "exports/raw/a.zip",
|
||||
"exports/extracted/a.md", "requests/a.json", "report-preview.md", "nested/fixture-secrets/not-excluded",
|
||||
];
|
||||
for (const path of paths) {
|
||||
await mkdir(dirname(join(run.root, path)), { recursive: true });
|
||||
await writeFile(join(run.root, path), `prefix ${canary} suffix`);
|
||||
}
|
||||
const findings = await scanSecrets({ runRoot: run.root, forbiddenValues: [canary], expectedGitRepositories: [] });
|
||||
assert.deepEqual(new Set(findings.map((finding) => finding.path)), new Set(paths));
|
||||
});
|
||||
|
||||
test("secret scanner examines reachable Git blobs, not just loose file bytes", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const canary = "GIT-CANARY-secret-value-987654";
|
||||
const gitRoot = join(run.root, "author");
|
||||
await mkdir(gitRoot);
|
||||
await execFileAsync("git", ["init", "--initial-branch=main"], { cwd: gitRoot });
|
||||
await execFileAsync("git", ["config", "user.name", "Scanner Test"], { cwd: gitRoot });
|
||||
await execFileAsync("git", ["config", "user.email", "scanner@example.invalid"], { cwd: gitRoot });
|
||||
await writeFile(join(gitRoot, "secret.txt"), canary);
|
||||
await execFileAsync("git", ["add", "secret.txt"], { cwd: gitRoot });
|
||||
await execFileAsync("git", ["commit", "-m", "secret blob"], { cwd: gitRoot });
|
||||
await execFileAsync("git", ["rm", "secret.txt"], { cwd: gitRoot });
|
||||
await execFileAsync("git", ["commit", "-m", "remove worktree copy"], { cwd: gitRoot });
|
||||
const findings = await scanSecrets({ runRoot: run.root, forbiddenValues: [canary], expectedGitRepositories: ["author"] });
|
||||
assert.equal(findings.some((finding) => finding.path.startsWith("git-object:")), true);
|
||||
});
|
||||
|
||||
test("successful lifecycle honors keep and cleanup", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const checks = exactScenarios();
|
||||
const kept = await runIntegration({ repositoryRoot, keep: true, checks });
|
||||
assert.equal(kept.exitCode, 0);
|
||||
assert.equal((await lstat(kept.runRoot)).isDirectory(), true);
|
||||
const cleaned = await runIntegration({ repositoryRoot, keep: false, checks });
|
||||
assert.equal(cleaned.exitCode, 0);
|
||||
await assert.rejects(lstat(cleaned.runRoot));
|
||||
});
|
||||
|
||||
test("command helper accepts only executable plus separate argv", async () => {
|
||||
await assert.rejects(runCommand("git status"));
|
||||
await assert.rejects(runCommand({ executable: "/bin/echo", argv: "hello" }));
|
||||
await assert.rejects(runCommand({ executable: "/bin/echo", argv: [], shell: true }));
|
||||
await assert.rejects(runCommand({ executable: "git status; rm -rf /", argv: [] }));
|
||||
await assert.rejects(runCommand({ executable: "/tmp/git", argv: ["--version"] }), /command executable is not allowlisted/);
|
||||
await assert.rejects(runCommand({ executable: "tht", argv: ["config", "check"] }), /command executable is invalid/);
|
||||
const scratchRoot = await fakeRepository();
|
||||
const executable = join(scratchRoot, "executable with spaces");
|
||||
await writeFile(executable, "#!/bin/sh\nprintf '%s' \"$1\"\n", { mode: 0o700 });
|
||||
await chmod(executable, 0o700);
|
||||
await assert.rejects(runCommand({ executable, argv: ["literal;not-a-shell"] }), /command executable is not allowlisted/);
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const { gitPath } = await resolveProductionExecutables({ repositoryRoot });
|
||||
const result = await runCommand({ executable: gitPath, argv: ["--version"] });
|
||||
assert.match(result.stdout, /^git version /);
|
||||
assert.equal(result.code, 0);
|
||||
});
|
||||
|
||||
test("raw runCommand rejects a configured clean filter before exact Git add", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const content = join(repositoryRoot, "workspace-content");
|
||||
const helper = join(repositoryRoot, "clean-helper");
|
||||
const marker = join(repositoryRoot, "clean-helper-ran");
|
||||
await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Fixture Curator"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-curator@example.invalid"], { cwd: repositoryRoot });
|
||||
await mkdir(content);
|
||||
await writeFile(join(content, "guide.md"), "content\n");
|
||||
await writeFile(join(repositoryRoot, ".gitattributes"), "workspace-content/** filter=bad\n");
|
||||
await writeFile(helper, `#!/bin/sh\nprintf ran > '${marker}'\ncat\n`, { mode: 0o700 });
|
||||
await execFileAsync("/usr/bin/git", ["config", "filter.bad.clean", `'${helper}'`], { cwd: repositoryRoot });
|
||||
|
||||
const { gitPath } = await resolveProductionExecutables({
|
||||
repositoryRoot: await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")),
|
||||
});
|
||||
await assert.rejects(
|
||||
runCommand({ executable: gitPath, argv: ["add", "workspace-content"], cwd: repositoryRoot, env: process.env }),
|
||||
/unsafe Git repository state/,
|
||||
);
|
||||
await assert.rejects(lstat(marker));
|
||||
});
|
||||
|
||||
test("raw runCommand rejects a diff driver textconv before exact Git show", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const marker = join(repositoryRoot, "textconv-helper-ran");
|
||||
const helper = join(repositoryRoot, "textconv-helper");
|
||||
await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Fixture Curator"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-curator@example.invalid"], { cwd: repositoryRoot });
|
||||
await writeFile(join(repositoryRoot, ".gitattributes"), "file diff=evil\n");
|
||||
await execFileAsync("/usr/bin/git", ["add", ".gitattributes"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["commit", "-m", "attributes"], { cwd: repositoryRoot });
|
||||
await writeFile(join(repositoryRoot, "file"), "v1\n");
|
||||
await execFileAsync("/usr/bin/git", ["add", "file"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["commit", "-m", "v1"], { cwd: repositoryRoot });
|
||||
await writeFile(join(repositoryRoot, "file"), "v2\n");
|
||||
await execFileAsync("/usr/bin/git", ["add", "file"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["commit", "-m", "v2"], { cwd: repositoryRoot });
|
||||
await writeFile(helper, `#!/bin/sh\nprintf ran > '${marker}'\nexit 0\n`, { mode: 0o700 });
|
||||
await execFileAsync("/usr/bin/git", ["config", "diff.evil.textconv", `'${helper}'`], { cwd: repositoryRoot });
|
||||
const emptyHooks = join(repositoryRoot, "registry", "locks", "empty-hooks");
|
||||
await mkdir(emptyHooks, { recursive: true });
|
||||
|
||||
const { gitPath } = await resolveProductionExecutables({
|
||||
repositoryRoot: await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")),
|
||||
});
|
||||
await assert.rejects(
|
||||
runCommand({ executable: gitPath, argv: ["-c", `core.hooksPath=${emptyHooks}`, "show", "HEAD"], cwd: repositoryRoot, env: process.env }),
|
||||
/unsafe Git repository state/,
|
||||
);
|
||||
await assert.rejects(lstat(marker));
|
||||
});
|
||||
|
||||
|
||||
test("safe environment rejects ambient THT and keeps only strict process allowlist plus fixture values", () => {
|
||||
const safe = buildSafeEnvironment({
|
||||
ambient: { PATH: "/safe/bin", HOME: "/home/test", LANG: "C", THT_SECRETS_FILE: "/real/secrets", AWS_SECRET_ACCESS_KEY: "real" },
|
||||
fixture: { THT_BIN: "/fixture/tht", THT_WORKSPACE_SECRET_ROOTS: "/fixture/secrets" },
|
||||
});
|
||||
assert.deepEqual(safe, {
|
||||
LANG: "C", THT_BIN: "/fixture/tht", THT_WORKSPACE_SECRET_ROOTS: "/fixture/secrets",
|
||||
});
|
||||
});
|
||||
|
||||
test("secret scan fails closed when Git enumeration fails", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await mkdir(join(run.root, "remote.git"));
|
||||
await assert.rejects(scanSecrets({ runRoot: run.root, forbiddenValues: ["CANARY-value-123"] }), /Git secret scan failed closed/);
|
||||
});
|
||||
|
||||
|
||||
test("negative request evidence persists only case label and expected input field", () => {
|
||||
const value = negativeRequestEvidence("credential-field", "evidence.source.password");
|
||||
assert.deepEqual(value, { case: "credential-field", expectedInputField: "evidence.source.password" });
|
||||
assert.equal(JSON.stringify(value).includes("body"), false);
|
||||
});
|
||||
|
||||
test("external fetch guard permits only the owned loopback API and records external attempts", async () => {
|
||||
const called = [];
|
||||
const guard = installExternalFetchGuard("http://127.0.0.1:12345", async (url) => { called.push(String(url)); return { ok: true }; });
|
||||
await guard.fetch("http://127.0.0.1:12345/workspaces");
|
||||
await assert.rejects(guard.fetch("https://evidence.example.test/guide.md"), /external fetch prohibited/);
|
||||
await assert.rejects(guard.fetch("http://127.0.0.1:9999/health"), /external fetch prohibited/);
|
||||
assert.deepEqual(called, ["http://127.0.0.1:12345/workspaces"]);
|
||||
assert.equal(guard.externalAttempts.length, 2);
|
||||
});
|
||||
|
||||
|
||||
test("export archive evidence path matches the persisted binary request id", () => {
|
||||
assert.equal(exportArchiveEvidencePath("export-p1-filesystem"), "exports/raw/export-p1-filesystem.zip");
|
||||
});
|
||||
|
||||
|
||||
test("announce callback observes PASS and manual pending before non-keep cleanup", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
let observed;
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: false, checks: exactScenarios(),
|
||||
announce: async ({ report, runRoot }) => {
|
||||
observed = { overall: report.overall, manual: "PENDING", rootExists: (await lstat(runRoot)).isDirectory() };
|
||||
},
|
||||
});
|
||||
assert.deepEqual(observed, { overall: "PASS", manual: "PENDING", rootExists: true });
|
||||
assert.equal(result.retained, false);
|
||||
});
|
||||
|
||||
test("public wrapper replaces ambient environment before invoking the runner", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /P1_ACCEPTANCE_FAIL_AT|LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /export THT_BIN/);
|
||||
});
|
||||
|
||||
|
||||
test("network guard is installed globally, rejects non-loopback sockets, and permits one owned listener", async () => {
|
||||
const server = createServer((socket) => socket.end("ok"));
|
||||
await new Promise((resolvePromise, reject) => server.listen(0, "127.0.0.1", (error) => error ? reject(error) : resolvePromise()));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === "object");
|
||||
const guard = installNetworkGuard();
|
||||
try {
|
||||
guard.addOwnedOrigin(`http://127.0.0.1:${address.port}`);
|
||||
const contents = await new Promise((resolvePromise, reject) => {
|
||||
const socket = connect({ host: "127.0.0.1", port: address.port });
|
||||
let value = "";
|
||||
socket.setEncoding("utf8");
|
||||
socket.on("data", (chunk) => { value += chunk; });
|
||||
socket.on("end", () => resolvePromise(value));
|
||||
socket.on("error", reject);
|
||||
});
|
||||
assert.equal(contents, "ok");
|
||||
assert.throws(() => connect({ host: "example.com", port: 80 }), /external network connection prohibited/);
|
||||
await assert.rejects(globalThis.fetch("https://example.com/"), /external network connection prohibited/);
|
||||
assert.equal(guard.externalAttempts.length, 2);
|
||||
} finally {
|
||||
guard.restore();
|
||||
await new Promise((resolvePromise) => server.close(resolvePromise));
|
||||
}
|
||||
});
|
||||
|
||||
test("report validation rejects duplicate artifact paths across checks", () => {
|
||||
const report = validReport();
|
||||
report.checks[1].artifacts[0].path = report.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(report), /report artifact path is duplicated/);
|
||||
});
|
||||
|
||||
test("virtual report leakage yields a minimal sanitized exact-15 FAIL report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const canary = "VIRTUAL-CANARY-12345678";
|
||||
const checks = exactScenarios(async (id) => ({
|
||||
commands: [],
|
||||
artifacts: id === CHECK_IDS[0] ? [{ path: `logs/${canary}.json`, sha256: "a".repeat(64) }] : [],
|
||||
}));
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
checks,
|
||||
setup: async (_run, _repositoryRoot, _env, ctx) => {
|
||||
ctx.forbiddenValues = [canary];
|
||||
return ctx;
|
||||
},
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
const bytes = await readFile(join(result.runRoot, "report.json"));
|
||||
assert.equal(bytes.includes(Buffer.from(canary)), false);
|
||||
const report = JSON.parse(bytes);
|
||||
assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS);
|
||||
assert(report.checks.every(({ status, commands, artifacts }) => status === "FAIL" && commands.length === 0 && artifacts.length === 0));
|
||||
});
|
||||
|
||||
test("partial setup preserves forbidden values and never writes secret-bearing report bytes", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const canary = "PARTIAL-SETUP-CANARY-12345678";
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
setup: async (run, _repositoryRoot, _env, ctx) => {
|
||||
ctx.forbiddenValues = [canary];
|
||||
await mkdir(join(run.root, "logs"), { recursive: true });
|
||||
await writeFile(join(run.root, "logs", "partial-setup.log"), canary);
|
||||
throw new Error(`unsafe ${canary}`);
|
||||
},
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
const bytes = await readFile(join(result.runRoot, "report.json"));
|
||||
assert.equal(bytes.includes(Buffer.from(canary)), false);
|
||||
const report = JSON.parse(bytes);
|
||||
assert.equal(report.checks.length, 15);
|
||||
assert(report.checks.every(({ status }) => status === "FAIL"));
|
||||
});
|
||||
|
||||
test("secret scan fails closed when either expected Git repository is missing", async () => {
|
||||
for (const missing of ["remote.git", "author"]) {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const present = missing === "remote.git" ? "author" : "remote.git";
|
||||
await mkdir(join(run.root, present));
|
||||
await execFileAsync("git", present === "remote.git" ? ["init", "--bare", join(run.root, present)] : ["init", join(run.root, present)]);
|
||||
await assert.rejects(scanSecrets({ runRoot: run.root, forbiddenValues: ["CANARY-value-123"] }), new RegExp(`missing expected Git repository: ${missing.replace(".", "\\.")}`));
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("runIntegration fails closed when a later duplicate overwrites stale artifact evidence", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const checks = exactScenarios(async (id) => {
|
||||
if (id === CHECK_IDS[0]) {
|
||||
await mkdir(join(repositoryRoot, ".artifacts", "p1-integration", "scratch"), { recursive: true });
|
||||
}
|
||||
return { commands: [], artifacts: [] };
|
||||
});
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: true,
|
||||
setup: async (run, _repositoryRoot, _env, ctx) => {
|
||||
const path = join(run.root, "logs", "overwritten.json");
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, "first");
|
||||
const stale = { path: "logs/overwritten.json", sha256: "a7937b64b8caa58f03721bb6bacf9e92a2c78987f5d1692a065a4698e006c4ca" };
|
||||
checks[0].run = async () => ({ commands: [], artifacts: [stale] });
|
||||
checks[1].run = async () => {
|
||||
await writeFile(path, "second");
|
||||
return { commands: [], artifacts: [{ path: stale.path, sha256: "16367aacb67a4a017c8da8ab95682ccb389c61bb315f3425e2f2666f2476d1ce" }] };
|
||||
};
|
||||
return ctx;
|
||||
},
|
||||
checks,
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.report.checks.length, 15);
|
||||
assert(result.report.checks.every(({ status, artifacts }) => status === "FAIL" && artifacts.length === 0));
|
||||
});
|
||||
|
||||
test("production surface guard rejects and records UDP, Worker, git ls-remote, and unexpected python", async () => {
|
||||
const runRoot = await fakeRepository();
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot });
|
||||
const guard = installProductionSurfaceGuard({
|
||||
...executables, runRoot, environment: { ...process.env }, originalFetch: globalThis.fetch,
|
||||
});
|
||||
try {
|
||||
assert.throws(() => dgram.createSocket("udp4"), /prohibited production surface/);
|
||||
assert.throws(() => new Worker("", { eval: true }), /prohibited production surface/);
|
||||
await assert.rejects(runCommand({ executable: executables.gitPath, argv: ["ls-remote", "https://example.com/repo.git"] }), /Git command is prohibited/);
|
||||
const childProcess = await import("node:child_process");
|
||||
assert.throws(() => childProcess.spawn(executables.pythonPath, ["-c", "print('unexpected')"]), /child command is prohibited/);
|
||||
assert.deepEqual(new Set(guard.events.filter(({ outcome }) => outcome === "REJECTED").map(({ surface }) => surface)),
|
||||
new Set(["dgram", "worker_threads", "child_process"]));
|
||||
} finally {
|
||||
guard.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test("listener close rejection retains listening truth and forces exact-15 FAIL", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const server = createServer();
|
||||
await new Promise((resolvePromise, reject) => server.listen(0, "127.0.0.1", (error) => error ? reject(error) : resolvePromise()));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === "object");
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: false, checks: exactScenarios(),
|
||||
setup: async (run, _repositoryRoot, _env, ctx) => {
|
||||
ctx.services = [{ name: "primary", baseUrl: `http://127.0.0.1:${address.port}`, app: { close: async () => { throw new Error("close rejected"); } } }];
|
||||
const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8"));
|
||||
value.listeners[0] = { name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0, actualPort: address.port, pid: process.pid, state: "listening" };
|
||||
await writeFile(join(run.root, "ownership.json"), `${JSON.stringify(value, null, 2)}\n`);
|
||||
return ctx;
|
||||
},
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const owner = JSON.parse(await readFile(join(result.runRoot, "ownership.json"), "utf8"));
|
||||
assert.notEqual(owner.listeners[0].state, "closed");
|
||||
assert(result.report.checks.every(({ status }) => status === "FAIL"));
|
||||
await new Promise((resolvePromise) => server.close(resolvePromise));
|
||||
});
|
||||
|
||||
test("ownership close write failure forces retained exact-15 FAIL", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const server = createServer();
|
||||
await new Promise((resolvePromise, reject) => server.listen(0, "127.0.0.1", (error) => error ? reject(error) : resolvePromise()));
|
||||
const address = server.address();
|
||||
assert(address && typeof address === "object");
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: false, checks: exactScenarios(),
|
||||
ownershipWriter: async (_run, update) => { if (update?.state === "closed") throw new Error("owned write rejected"); },
|
||||
setup: async (_run, _repositoryRoot, _env, ctx) => {
|
||||
ctx.services = [{ name: "primary", baseUrl: `http://127.0.0.1:${address.port}`, app: { close: async () => await new Promise((resolvePromise) => server.close(resolvePromise)) } }];
|
||||
return ctx;
|
||||
},
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
assert(result.report.checks.every(({ status }) => status === "FAIL"));
|
||||
});
|
||||
|
||||
test("nested runIntegration is rejected before process-global mutation and outer restoration remains owned", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalPath = process.env.PATH;
|
||||
let nestedError;
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: true, checks: exactScenarios(),
|
||||
setup: async (_run, _repositoryRoot, _env, ctx) => {
|
||||
try { await runIntegration({ repositoryRoot, keep: true, checks: exactScenarios() }); } catch (error) { nestedError = error; }
|
||||
assert.equal(globalThis.fetch, originalFetch);
|
||||
assert.equal(process.env.PATH, originalPath);
|
||||
return ctx;
|
||||
},
|
||||
});
|
||||
assert.match(nestedError?.message ?? "", /already active/);
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(globalThis.fetch, originalFetch);
|
||||
assert.equal(process.env.PATH, originalPath);
|
||||
});
|
||||
|
||||
|
||||
test("environment tampering fails the audit and restores the caller environment", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const before = { ...process.env };
|
||||
const checks = exactScenarios(async (id) => {
|
||||
if (id === CHECK_IDS[0]) process.env.P1_ACCEPTANCE_UNOWNED = "tampered";
|
||||
return { commands: [], artifacts: [] };
|
||||
});
|
||||
const result = await runIntegration({
|
||||
repositoryRoot, keep: true, checks,
|
||||
setup: async (_run, _repositoryRoot, _env, ctx) => { ctx.env = { P1_ACCEPTANCE_OWNED: "yes" }; return ctx; },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert(result.report.checks.every(({ status }) => status === "FAIL"));
|
||||
assert.deepEqual({ ...process.env }, before);
|
||||
});
|
||||
|
||||
test("production guard detects global tampering and restores without stranding patches", async () => {
|
||||
const runRoot = await fakeRepository();
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot });
|
||||
const originalFetch = globalThis.fetch;
|
||||
const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env }, originalFetch });
|
||||
globalThis.fetch = originalFetch;
|
||||
assert.throws(() => guard.restore(), /ownership restoration failed/);
|
||||
assert.equal(globalThis.fetch, originalFetch);
|
||||
const childProcess = await import("node:child_process");
|
||||
assert.doesNotThrow(() => childProcess.spawn);
|
||||
});
|
||||
|
||||
|
||||
test("Git grammar rejects helper, config, alias, and network-capable spellings with one event each", async () => {
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const runRoot = await fakeRepository();
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot, thtBin: "/tmp/hostile-tht" });
|
||||
const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } });
|
||||
const source = join(runRoot, "source.git");
|
||||
const destination = join(runRoot, "destination");
|
||||
const marker = join(runRoot, "helper-ran");
|
||||
await execFileAsync(executables.gitPath, ["init", "--bare", source]);
|
||||
const helper = join(runRoot, "upload-helper");
|
||||
await writeFile(helper, `#!/bin/sh\nprintf ran > "${marker}"\nexit 99\n`, { mode: 0o700 });
|
||||
const prohibited = [
|
||||
["clone", `--upload-pack=${helper}`, source, destination],
|
||||
["clone", "--receive-pack=/tmp/helper", source, destination],
|
||||
["--exec-path=/tmp", "status"],
|
||||
["-c", "alias.status=!touch /tmp/pwn", "status"],
|
||||
["-c", "core.hooksPath=/tmp/hooks", "status"],
|
||||
["-c", "diff.external=/tmp/helper", "status"],
|
||||
["config", "filter.bad.clean", "/tmp/helper"],
|
||||
["ls-remote", "https://example.com/repo.git"],
|
||||
];
|
||||
try {
|
||||
for (const argv of prohibited) {
|
||||
const before = guard.events.length;
|
||||
await assert.rejects(runCommand({ executable: executables.gitPath, argv }), /Git command is prohibited/);
|
||||
assert.equal(guard.events.length - before, 1);
|
||||
assert.equal(guard.events.at(-1).outcome, "REJECTED");
|
||||
}
|
||||
await assert.rejects(lstat(marker));
|
||||
} finally { guard.restore(); }
|
||||
});
|
||||
|
||||
test("production executables ignore ambient THT and bind the generated tht entrypoint to reviewed source", async () => {
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const hostile = join(await fakeRepository(), "tht");
|
||||
await writeFile(hostile, "#!/bin/sh\nexit 0\n", { mode: 0o700 });
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot, thtBin: hostile });
|
||||
assert.equal(executables.thtPath, join(repositoryRoot, "harness", ".venv", "bin", "tht"));
|
||||
assert.equal(executables.thtIdentity.sourceRoot, join(repositoryRoot, "harness", "tht"));
|
||||
assert.equal(executables.thtIdentity.sourceStatus, "git-index-byte-identical");
|
||||
assert.equal(executables.thtIdentity.entrypoint, "generated-console-script");
|
||||
assert.match(executables.thtIdentity.pythonPath, /python3(?:\.\d+)?$/);
|
||||
});
|
||||
|
||||
test("tht accepts only config check for one owned rendered yaml", async () => {
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const runRoot = await fakeRepository();
|
||||
const rendered = join(runRoot, "rendered", "workspace.yaml");
|
||||
await mkdir(dirname(rendered), { recursive: true });
|
||||
await writeFile(rendered, "profile: acceptance\n");
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot });
|
||||
const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } });
|
||||
const childProcess = await import("node:child_process");
|
||||
try {
|
||||
for (const argv of [
|
||||
["config", "check"], ["config", "check", "-c", "/tmp/unowned.yaml"],
|
||||
["doctor"], ["config", "check", "-c", rendered, "--extra"],
|
||||
]) {
|
||||
const before = guard.events.length;
|
||||
assert.throws(() => childProcess.execFile(executables.thtPath, argv), /THT command is prohibited/);
|
||||
assert.equal(guard.events.length - before, 1);
|
||||
}
|
||||
} finally { guard.restore(); }
|
||||
});
|
||||
|
||||
test("production guard installation rolls back every patch and owner on every injected patch failure", async () => {
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const runRoot = await fakeRepository();
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot });
|
||||
const childProcess = await import("node:child_process");
|
||||
const originalSpawn = childProcess.spawn;
|
||||
const originalDgram = dgram.createSocket;
|
||||
const originalFetch = globalThis.fetch;
|
||||
for (let failPatchAt = 1; failPatchAt <= 12; failPatchAt += 1) {
|
||||
assert.throws(() => installProductionSurfaceGuard({
|
||||
...executables, runRoot, environment: { ...process.env }, failPatchAt,
|
||||
}), /injected production patch failure/);
|
||||
assert.equal(childProcess.spawn, originalSpawn);
|
||||
assert.equal(dgram.createSocket, originalDgram);
|
||||
assert.equal(globalThis.fetch, originalFetch);
|
||||
const reacquired = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } });
|
||||
reacquired.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test("command bounds reject zero, negative, fractional, and nonnumeric timeouts with one sanitized event", async () => {
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const runRoot = await fakeRepository();
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot });
|
||||
const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } });
|
||||
try {
|
||||
for (const timeoutMs of [0, -1, 1.5, NaN]) {
|
||||
const before = guard.events.length;
|
||||
await assert.rejects(runCommand({ executable: executables.gitPath, argv: ["--version"], timeoutMs }), /command bounds are invalid/);
|
||||
assert.equal(guard.events.length - before, 1);
|
||||
}
|
||||
} finally { guard.restore(); }
|
||||
});
|
||||
|
||||
test("public wrapper has no ambient command resolution and isolates the build and runner", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh"), "utf8");
|
||||
assert.doesNotMatch(wrapper, /command\s+-v/);
|
||||
assert.doesNotMatch(wrapper, /\b(?:node|npm)\s+--prefix/);
|
||||
assert.match(wrapper, /env -i/);
|
||||
assert.match(wrapper, /npm-cli\.js/);
|
||||
assert.match(wrapper, /"\$node_path" "\$npm_path"/);
|
||||
assert.match(wrapper, /\/bin\/rm -rf -- "\$repo_root\/backend\/dist"/);
|
||||
});
|
||||
|
||||
|
||||
test("hostile PATH Node npm and THT substitutes never execute at the public wrapper boundary", async () => {
|
||||
const hostileRoot = await fakeRepository();
|
||||
const marker = join(hostileRoot, "ambient-tool-ran");
|
||||
for (const name of ["node", "npm", "tht"]) {
|
||||
const path = join(hostileRoot, name);
|
||||
await writeFile(path, `#!/bin/sh\nprintf '%s' '${name}' >> '${marker}'\nexit 97\n`, { mode: 0o700 });
|
||||
await chmod(path, 0o700);
|
||||
}
|
||||
const wrapper = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh");
|
||||
await assert.rejects(execFileAsync(wrapper, ["invalid"], {
|
||||
env: { ...process.env, PATH: hostileRoot, THT_BIN: join(hostileRoot, "tht") }, timeout: 30_000,
|
||||
}));
|
||||
await assert.rejects(lstat(marker));
|
||||
});
|
||||
|
||||
|
||||
async function fakeTrustedThtRepository() {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const realRepository = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const harness = join(repositoryRoot, "harness");
|
||||
const sourceRoot = join(harness, "tht");
|
||||
await mkdir(harness, { recursive: true });
|
||||
await cp(join(realRepository, "harness", "tht"), sourceRoot, {
|
||||
recursive: true, filter: (path) => !path.split("/").includes("__pycache__") && !path.endsWith(".pyc"),
|
||||
});
|
||||
await cp(join(realRepository, "harness", "pyproject.toml"), join(harness, "pyproject.toml"));
|
||||
const realExecutables = await resolveProductionExecutables({ repositoryRoot: realRepository });
|
||||
const pythonName = realExecutables.thtIdentity.pythonPath.split("/").at(-1);
|
||||
const venvBin = join(harness, ".venv", "bin");
|
||||
const sitePackages = join(harness, ".venv", "lib", pythonName, "site-packages");
|
||||
await mkdir(venvBin, { recursive: true });
|
||||
await mkdir(sitePackages, { recursive: true });
|
||||
await symlink(realExecutables.thtIdentity.pythonCanonicalPath, join(venvBin, "python"));
|
||||
await symlink(realExecutables.thtIdentity.pythonCanonicalPath, join(venvBin, pythonName));
|
||||
const entrypoint = `#!${join(venvBin, pythonName)}\nimport sys\nfrom tht.cli import app\nif __name__ == '__main__':\n if sys.argv[0].endswith('.exe'):\n sys.argv[0] = sys.argv[0][:-4]\n sys.exit(app())\n`;
|
||||
await writeFile(join(venvBin, "tht"), entrypoint, { mode: 0o700 });
|
||||
const realSite = join(realRepository, "harness", ".venv", "lib", pythonName, "site-packages");
|
||||
const realFinderName = (await import("node:fs/promises")).readdir(realSite).then((entries) => entries.find((name) => /^__editable___tht_.*_finder\.py$/.test(name)));
|
||||
const finderName = await realFinderName;
|
||||
const realFinder = await readFile(join(realSite, finderName), "utf8");
|
||||
const finder = realFinder.replaceAll(join(realRepository, "harness", "tht"), sourceRoot);
|
||||
await writeFile(join(sitePackages, finderName), finder);
|
||||
const moduleName = finderName.slice(0, -3);
|
||||
await writeFile(join(sitePackages, "__editable__.tht-0.1.0.pth"), `import ${moduleName}; ${moduleName}.install()`);
|
||||
await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Test"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-test@example.invalid"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["add", "harness/tht", "harness/pyproject.toml"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["commit", "-m", "trusted source"], { cwd: repositoryRoot });
|
||||
return { repositoryRoot, sourceRoot, sitePackages, finderName };
|
||||
}
|
||||
|
||||
test("Git rejects configured upload-pack, clean filter, and hook state before exact allowed operations", async () => {
|
||||
const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
|
||||
const runRoot = await fakeRepository();
|
||||
const remote = join(runRoot, "remote.git");
|
||||
const author = join(runRoot, "author");
|
||||
await execFileAsync("/usr/bin/git", ["init", "--bare", "--initial-branch=main", remote]);
|
||||
await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main", author]);
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Fixture Curator"], { cwd: author });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-curator@example.invalid"], { cwd: author });
|
||||
await writeFile(join(author, "seed"), "seed\n");
|
||||
await execFileAsync("/usr/bin/git", ["add", "seed"], { cwd: author });
|
||||
await execFileAsync("/usr/bin/git", ["commit", "-m", "seed"], { cwd: author });
|
||||
await execFileAsync("/usr/bin/git", ["remote", "add", "origin", remote], { cwd: author });
|
||||
await execFileAsync("/usr/bin/git", ["push", "origin", "main"], { cwd: author });
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot });
|
||||
const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } });
|
||||
try {
|
||||
for (const [kind, configure, argv] of [
|
||||
["upload", async (helper) => execFileAsync("/usr/bin/git", ["config", "remote.origin.uploadpack", helper], { cwd: author }), ["fetch", "origin", "main"]],
|
||||
["filter", async (helper) => {
|
||||
await mkdir(join(author, "workspace-content"), { recursive: true });
|
||||
await writeFile(join(author, ".gitattributes"), "workspace-content/** filter=bad\n");
|
||||
await execFileAsync("/usr/bin/git", ["config", "filter.bad.clean", helper], { cwd: author });
|
||||
}, ["add", "workspace-content"]],
|
||||
["hook", async (helper) => { await cp(helper, join(author, ".git", "hooks", "pre-commit")); }, ["commit", "-m", "Bootstrap curated P1 content"]],
|
||||
]) {
|
||||
await execFileAsync("/usr/bin/git", ["config", "--unset-all", "remote.origin.uploadpack"], { cwd: author }).catch(() => {});
|
||||
await execFileAsync("/usr/bin/git", ["config", "--remove-section", "filter.bad"], { cwd: author }).catch(() => {});
|
||||
await rm(join(author, ".gitattributes"), { force: true });
|
||||
await rm(join(author, ".git", "hooks", "pre-commit"), { force: true });
|
||||
const marker = join(runRoot, `${kind}-marker`);
|
||||
const helper = join(runRoot, `${kind}-helper`);
|
||||
await writeFile(helper, `#!/bin/sh\nprintf ran > '${marker}'\nexec /usr/bin/git-upload-pack \"$@\"\n`, { mode: 0o700 });
|
||||
await configure(helper);
|
||||
const before = guard.events.length;
|
||||
await assert.rejects(runCommand({ executable: executables.gitPath, argv, cwd: author, env: { ...process.env } }), /unsafe Git repository state/);
|
||||
assert.equal(guard.events.length - before, 1);
|
||||
assert.equal(guard.events.at(-1).outcome, "REJECTED");
|
||||
await assert.rejects(lstat(marker));
|
||||
}
|
||||
} finally { guard.restore(); }
|
||||
});
|
||||
|
||||
test("trusted tht rejects executable finder code and Git-hidden source changes", async () => {
|
||||
const maliciousFinder = await fakeTrustedThtRepository();
|
||||
const finderPath = join(maliciousFinder.sitePackages, maliciousFinder.finderName);
|
||||
await writeFile(finderPath, `open('${join(maliciousFinder.repositoryRoot, "finder-marker")}', 'w').write('ran')\n${await readFile(finderPath, "utf8")}`);
|
||||
await assert.rejects(resolveProductionExecutables({ repositoryRoot: maliciousFinder.repositoryRoot }), /editable binding is invalid/);
|
||||
|
||||
const ignoredPyc = await fakeTrustedThtRepository();
|
||||
await mkdir(join(ignoredPyc.sitePackages, "__pycache__"));
|
||||
await writeFile(join(ignoredPyc.sitePackages, "__pycache__", `${ignoredPyc.finderName.slice(0, -3)}.cpython-313.pyc`), "malicious bytecode");
|
||||
await assert.rejects(resolveProductionExecutables({ repositoryRoot: ignoredPyc.repositoryRoot }), /import startup override/);
|
||||
|
||||
const hiddenSource = await fakeTrustedThtRepository();
|
||||
const sourcePath = join(hiddenSource.sourceRoot, "cli", "__init__.py");
|
||||
await execFileAsync("/usr/bin/git", ["update-index", "--assume-unchanged", "harness/tht/cli/__init__.py"], { cwd: hiddenSource.repositoryRoot });
|
||||
await writeFile(sourcePath, `${await readFile(sourcePath, "utf8")}\n# malicious hidden swap\n`);
|
||||
await assert.rejects(resolveProductionExecutables({ repositoryRoot: hiddenSource.repositoryRoot }), /source bytes differ from Git/);
|
||||
});
|
||||
|
||||
test("trusted tht guard rejects and records post-resolution entrypoint finder and source swaps at spawn", async () => {
|
||||
for (const target of ["entrypoint", "finder", "source"]) {
|
||||
const fixture = await fakeTrustedThtRepository();
|
||||
const executables = await resolveProductionExecutables({ repositoryRoot: fixture.repositoryRoot });
|
||||
const runRoot = await fakeRepository();
|
||||
const configPath = join(runRoot, "rendered", "workspace.yaml");
|
||||
await mkdir(dirname(configPath), { recursive: true });
|
||||
await writeFile(configPath, "profile: acceptance\n");
|
||||
const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } });
|
||||
try {
|
||||
const path = target === "entrypoint" ? executables.thtPath
|
||||
: target === "finder" ? join(fixture.sitePackages, fixture.finderName)
|
||||
: join(fixture.sourceRoot, "cli", "__init__.py");
|
||||
await writeFile(path, `${await readFile(path, "utf8")}\n# post-resolution swap\n`, target === "entrypoint" ? { mode: 0o700 } : undefined);
|
||||
const childProcess = await import("node:child_process");
|
||||
assert.throws(() => childProcess.execFile(executables.thtPath, ["config", "check", "-c", configPath], {
|
||||
cwd: join(fixture.repositoryRoot, "harness"), env: { ...process.env },
|
||||
}), /trusted THT identity changed/);
|
||||
assert.equal(guard.events.at(-1).outcome, "REJECTED");
|
||||
} finally { guard.restore(); }
|
||||
}
|
||||
});
|
||||
|
||||
test("secret scan fails closed on a recoverable symlink outside fixture-secrets", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const canary = "CANARY-symlink-secret-123456";
|
||||
await mkdir(join(run.root, "fixture-secrets"));
|
||||
await writeFile(join(run.root, "fixture-secrets", "token"), canary);
|
||||
await mkdir(join(run.root, "responses"));
|
||||
await symlink(join(run.root, "fixture-secrets", "token"), join(run.root, "responses", "leak"));
|
||||
await assert.rejects(scanSecrets({ runRoot: run.root, forbiddenValues: [canary], expectedGitRepositories: [] }), /symlink outside fixture-secrets/);
|
||||
});
|
||||
|
||||
test("direct public wrapper clears startup files and exported functions before Bash starts", async () => {
|
||||
const root = await fakeRepository();
|
||||
const bashStartup = join(root, "bash-startup");
|
||||
const envStartup = join(root, "env-startup");
|
||||
const bashMarker = join(root, "bash-env-ran");
|
||||
const envMarker = join(root, "env-ran");
|
||||
const functionMarker = join(root, "exported-function-ran");
|
||||
await writeFile(bashStartup, `printf sourced > '${bashMarker}'\n`);
|
||||
await writeFile(envStartup, `printf sourced > '${envMarker}'\n`);
|
||||
const wrapper = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh");
|
||||
await assert.rejects(execFileAsync(wrapper, ["invalid"], {
|
||||
env: {
|
||||
...process.env,
|
||||
BASH_ENV: bashStartup,
|
||||
ENV: envStartup,
|
||||
"BASH_FUNC_cd%%": `() { printf function > '${functionMarker}'; builtin cd "$@"; }`,
|
||||
},
|
||||
}));
|
||||
for (const marker of [bashMarker, envMarker, functionMarker]) await assert.rejects(lstat(marker));
|
||||
assert.match(await readFile(wrapper, "utf8"), /^#!\/usr\/bin\/env -S -i PATH=\/usr\/bin:\/bin \/bin\/bash\n/);
|
||||
});
|
||||
|
||||
test("final listener ownership state is a declared hash-bound report artifact", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, checks: exactScenarios() });
|
||||
const artifact = result.report.checks.flatMap(({ artifacts }) => artifacts).find(({ path }) => path === "logs/final-ownership.json");
|
||||
assert(artifact);
|
||||
const bytes = await readFile(join(result.runRoot, artifact.path));
|
||||
const { createHash } = await import("node:crypto");
|
||||
assert.equal(createHash("sha256").update(bytes).digest("hex"), artifact.sha256);
|
||||
const value = JSON.parse(bytes);
|
||||
assert.deepEqual(value.listeners.map(({ state }) => state), ["not_started", "not_started"]);
|
||||
});
|
||||
|
||||
test("repository provenance binds clean HEAD tree and backend source/dist manifests and rejects dirty state", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
await mkdir(join(repositoryRoot, "backend", "src"), { recursive: true });
|
||||
await mkdir(join(repositoryRoot, "backend", "scripts"), { recursive: true });
|
||||
await mkdir(join(repositoryRoot, "backend", "dist"), { recursive: true });
|
||||
await writeFile(join(repositoryRoot, "backend", "src", "app.ts"), "export const value = 1;\n");
|
||||
await writeFile(join(repositoryRoot, "backend", "scripts", "p1-acceptance.mjs"), "export {};\n");
|
||||
await writeFile(join(repositoryRoot, "backend", "dist", "app.js"), "export const value = 1;\n");
|
||||
await writeFile(join(repositoryRoot, "backend", "package.json"), "{}\n");
|
||||
await writeFile(join(repositoryRoot, "backend", "package-lock.json"), "{}\n");
|
||||
await writeFile(join(repositoryRoot, "backend", "tsconfig.json"), "{}\n");
|
||||
await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Test"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-test@example.invalid"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["add", "backend"], { cwd: repositoryRoot });
|
||||
await execFileAsync("/usr/bin/git", ["commit", "-m", "clean tree"], { cwd: repositoryRoot });
|
||||
const provenance = await collectRepositoryProvenance({ repositoryRoot, gitPath: "/usr/bin/git" });
|
||||
assert.match(provenance.head, /^[0-9a-f]{40}$/);
|
||||
assert.match(provenance.tree, /^[0-9a-f]{40}$/);
|
||||
assert.equal(provenance.clean, true);
|
||||
assert.equal(provenance.backendSource.files.some(({ path }) => path === "src/app.ts"), true);
|
||||
assert.equal(provenance.backendDist.files.some(({ path }) => path === "dist/app.js"), true);
|
||||
await writeFile(join(repositoryRoot, "backend", "src", "app.ts"), "dirty\n");
|
||||
await assert.rejects(collectRepositoryProvenance({ repositoryRoot, gitPath: "/usr/bin/git" }), /repository is not clean/);
|
||||
});
|
||||
@@ -1,609 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFile, spawn } from "node:child_process";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { closeSync, constants, fstatSync, fsyncSync, lstatSync, openSync, realpathSync } from "node:fs";
|
||||
import { access, chmod, lstat, mkdir, open, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import net from "node:net";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const exec = promisify(execFile); const modulePath=fileURLToPath(import.meta.url); const defaultRepositoryRoot=realpathSync(resolve(dirname(modulePath),"../.."));
|
||||
const HEX64=/^[0-9a-f]{64}$/; const PORT=8791; const HOST="127.0.0.1";
|
||||
export function fixedManualRoot(repositoryRoot=defaultRepositoryRoot){return join(realpathSync(repositoryRoot),".artifacts","manual-acceptance","p1");}
|
||||
function below(parent,child){const rel=relative(parent,child);return rel!==""&&!rel.startsWith(`..${sep}`)&&rel!==".."&&!isAbsolute(rel);}
|
||||
function noSymlinkExisting(repo,target){const rel=relative(repo,target);if(rel.startsWith("..")||isAbsolute(rel))throw new Error("root leaves repository");let cursor=repo;for(const part of rel.split(sep).filter(Boolean)){cursor=join(cursor,part);try{if(lstatSync(cursor).isSymbolicLink())throw new Error("owned root ancestor is a symlink");}catch(error){if(error.code==="ENOENT")break;throw error;}}}
|
||||
async function atomicWrite(path,bytes,mode=0o600){await mkdir(dirname(path),{recursive:true});const staging=join(dirname(path),`.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`);let h;try{h=await open(staging,"wx",mode);await h.writeFile(bytes);await h.sync();await h.close();h=undefined;await rename(staging,path);const fd=openSync(dirname(path),constants.O_RDONLY);try{fsyncSync(fd);}finally{closeSync(fd);}}finally{if(h)await h.close().catch(()=>{});await rm(staging,{force:true}).catch(()=>{});}}
|
||||
function directorySync(path){const fd=openSync(path,constants.O_RDONLY);try{fsyncSync(fd);}finally{closeSync(fd);}}
|
||||
async function exclusiveRecord(path,value,label){const bytes=`${JSON.stringify(value,null,2)}\n`;let handle,createdEntry;try{handle=await open(path,"wx",0o600);createdEntry=await handle.stat();await handle.chmod(0o600);await handle.writeFile(bytes);await handle.sync();await handle.close();handle=undefined;directorySync(dirname(path));return{path,bytes,dev:createdEntry.dev,ino:createdEntry.ino};}catch(error){if(handle)await handle.close().catch(()=>{});if(createdEntry)try{const current=await lstat(path);if(current.dev===createdEntry.dev&¤t.ino===createdEntry.ino)await rm(path);}catch{}if(error.code==="EEXIST")throw new Error(`${label} already exists; operator inspection required`);throw error;}}
|
||||
async function requireExactRecord(record){const entry=await lstat(record.path);if(!entry.isFile()||entry.isSymbolicLink()||(entry.mode&0o777)!==0o600||(record.dev!==undefined&&(entry.dev!==record.dev||entry.ino!==record.ino)))throw new Error("owned lifecycle record is unsafe");if(await readFile(record.path,"utf8")!==record.bytes)throw new Error("owned lifecycle record changed; operator inspection required");const after=await lstat(record.path);if(after.dev!==entry.dev||after.ino!==entry.ino)throw new Error("owned lifecycle record changed; operator inspection required");return after;}
|
||||
async function removeExactRecord(record){await requireExactRecord(record);await requireExactRecord(record);await rm(record.path);directorySync(dirname(record.path));}
|
||||
async function replaceExactRecord(record,value){await requireExactRecord(record);const bytes=`${JSON.stringify(value,null,2)}\n`,staging=join(dirname(record.path),`.${basename(record.path)}.${randomBytes(12).toString("hex")}.tmp`);let handle;try{handle=await open(staging,"wx",0o600);await handle.chmod(0o600);await handle.writeFile(bytes);await handle.sync();await handle.close();handle=undefined;await requireExactRecord(record);await rename(staging,record.path);const entry=await lstat(record.path);directorySync(dirname(record.path));return{path:record.path,bytes,dev:entry.dev,ino:entry.ino};}finally{if(handle)await handle.close().catch(()=>{});await rm(staging,{force:true}).catch(()=>{});}}
|
||||
function sameEntry(actual,expected){return actual.dev===expected.dev&&actual.ino===expected.ino;}
|
||||
async function requirePathIdentity(path,expected,label){let entry;try{entry=await lstat(path);}catch{throw new Error(`${label} identity changed`);}if(entry.isSymbolicLink()||!sameEntry(entry,expected))throw new Error(`${label} identity changed`);return entry;}
|
||||
async function acquireLifecycle(repo,operation){
|
||||
const lockPath=join(repo,".p1-manual-acceptance.lifecycle.lock"),artifacts=join(repo,".artifacts"),manualParent=join(artifacts,"manual-acceptance"),root=fixedManualRoot(repo);
|
||||
noSymlinkExisting(repo,manualParent);await mkdir(manualParent,{recursive:true,mode:0o700});noSymlinkExisting(repo,manualParent);
|
||||
const repoEntry=await lstat(repo),artifactsEntry=await lstat(artifacts),parentEntry=await lstat(manualParent);
|
||||
if(!repoEntry.isDirectory()||!artifactsEntry.isDirectory()||!parentEntry.isDirectory())throw new Error("lifecycle namespace identity is unsafe");
|
||||
const lifecycleNonce=randomBytes(32).toString("hex"),record=await exclusiveRecord(lockPath,{schemaVersion:1,kind:"p1-manual-lifecycle",operation,lifecycleNonce,root,repositoryRoot:repo},"external lifecycle lock"),entry=await requireExactRecord(record);
|
||||
return{...record,dev:entry.dev,ino:entry.ino,repoPath:repo,repoEntry,artifactsPath:artifacts,artifactsEntry,parentPath:manualParent,parentEntry,rootEntry:undefined};
|
||||
}
|
||||
async function requireLifecycleContext(lifecycle,{root=false}={}){
|
||||
await requireExactRecord(lifecycle);await requirePathIdentity(lifecycle.repoPath,lifecycle.repoEntry,"repository root");await requirePathIdentity(lifecycle.artifactsPath,lifecycle.artifactsEntry,"artifact root");await requirePathIdentity(lifecycle.parentPath,lifecycle.parentEntry,"manual acceptance parent");
|
||||
if(root&&lifecycle.rootEntry)await requirePathIdentity(join(lifecycle.parentPath,"p1"),lifecycle.rootEntry,"manual acceptance root");
|
||||
}
|
||||
async function bindLifecycleRoot(lifecycle,root){const entry=await lstat(root);if(!entry.isDirectory()||entry.isSymbolicLink())throw new Error("manual acceptance root identity is unsafe");lifecycle.rootEntry=entry;await requireLifecycleContext(lifecycle,{root:true});return entry;}
|
||||
async function findRootByIdentity(repo,identity){
|
||||
const artifacts=join(repo,".artifacts");let count=0;
|
||||
for(const parent of await readdir(artifacts,{withFileTypes:true})){if(++count>1024)throw new Error("manual cleanup search bound exceeded");if(!parent.isDirectory()||parent.isSymbolicLink())continue;const candidate=join(artifacts,parent.name,"p1");try{const entry=await lstat(candidate);if(entry.isDirectory()&&!entry.isSymbolicLink()&&sameEntry(entry,identity))return candidate;}catch{}
|
||||
}return undefined;
|
||||
}
|
||||
async function cleanupFailedPrepare(repo,lifecycle){if(!lifecycle.rootEntry)return;const candidate=await findRootByIdentity(repo,lifecycle.rootEntry);if(!candidate)return;const entry=await lstat(candidate);if(!sameEntry(entry,lifecycle.rootEntry)||entry.isSymbolicLink())throw new Error("failed prepare root identity changed");await rm(candidate,{recursive:true});}
|
||||
function legacySupervisorPath(root){return join(root,"installation/runtime/p1-backend-supervisor.mjs");}
|
||||
const CONTROL_PORT=8792;
|
||||
const PRELOAD_SOURCE=`import net from "node:net";
|
||||
import { createHash } from "node:crypto";
|
||||
import { closeSync, constants, fstatSync, openSync, readFileSync, readSync, realpathSync } from "node:fs";
|
||||
import { registerHooks } from "node:module";
|
||||
import { dirname, join, sep } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
const HOST="127.0.0.1",PORT=8792,HTTP_PORT=8791,HEX=/^[0-9a-f]{64}$/;
|
||||
const argv=process.argv.slice(2),noncePrefix="--p1-manual-nonce=",rootPrefix="--p1-root=",controlPrefix="--p1-control-nonce=",shaPrefix="--p1-entry-sha256=",devPrefix="--p1-entry-dev=",inoPrefix="--p1-entry-ino=";
|
||||
const prefixes=[noncePrefix,rootPrefix,controlPrefix,shaPrefix,devPrefix,inoPrefix];
|
||||
if(argv.length!==6||argv.some((value,index)=>!value.startsWith(prefixes[index])))throw new Error("manual control identity arguments refused");
|
||||
const nonce=argv[0].slice(noncePrefix.length),root=argv[1].slice(rootPrefix.length),controlNonce=argv[2].slice(controlPrefix.length),entrySha=argv[3].slice(shaPrefix.length),entryDev=argv[4].slice(devPrefix.length),entryIno=argv[5].slice(inoPrefix.length);
|
||||
if(!HEX.test(nonce)||!root.startsWith("/")||!HEX.test(controlNonce)||!HEX.test(entrySha)||!/^[0-9]+$/.test(entryDev)||!/^[0-9]+$/.test(entryIno))throw new Error("manual control identity refused");
|
||||
if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("manual distribution no-follow protection is unavailable");
|
||||
const entryStat=fstatSync(3),entrySource=readFileSync(3);if(!entryStat.isFile()||String(entryStat.dev)!==entryDev||String(entryStat.ino)!==entryIno||createHash("sha256").update(entrySource).digest("hex")!==entrySha)throw new Error("manual entrypoint FD identity refused");
|
||||
const manifestStat=fstatSync(4);if(!manifestStat.isFile()||manifestStat.size<1||manifestStat.size>8388608)throw new Error("manual distribution manifest FD identity refused");
|
||||
let manifest;try{manifest=JSON.parse(readFileSync(4));}catch{throw new Error("manual distribution manifest is malformed");}
|
||||
const entryPath=realpathSync(process.argv[1]),distRoot=dirname(entryPath);
|
||||
if(manifest?.schemaVersion!==1||manifest.kind!=="p1-manual-dist-manifest"||manifest.root!==distRoot||!manifest.files||typeof manifest.files!=="object"||Array.isArray(manifest.files))throw new Error("manual distribution manifest identity refused");
|
||||
const distEntries=Object.entries(manifest.files);if(distEntries.length<1||distEntries.length>20000)throw new Error("manual distribution manifest identity refused");
|
||||
const distBytes=new Map();
|
||||
for(const[rel,file]of distEntries){
|
||||
if(typeof rel!=="string"||!rel||rel.startsWith("/")||rel.startsWith("..")||rel.includes("\\\\")||rel.includes("/./")||rel.endsWith("/")||!Number.isSafeInteger(file?.size)||file.size<1||file.size>33554432||!HEX.test(file?.sha256??"")||!/^[0-9]+$/.test(String(file?.dev))||!/^[0-9]+$/.test(String(file?.ino)))throw new Error("manual distribution manifest is malformed");
|
||||
const path=join(distRoot,rel),fd=openSync(path,constants.O_RDONLY|constants.O_NOFOLLOW);
|
||||
try{
|
||||
const before=fstatSync(fd);
|
||||
if(!before.isFile()||before.nlink!==1||String(before.dev)!==String(file.dev)||String(before.ino)!==String(file.ino)||before.size!==file.size)throw new Error("manual distribution module identity changed");
|
||||
const bytes=Buffer.alloc(before.size);let offset=0;
|
||||
while(offset<bytes.length){const n=readSync(fd,bytes,offset,bytes.length-offset,offset);if(n<1)throw new Error("manual distribution module changed while binding");offset+=n;}
|
||||
const after=fstatSync(fd);
|
||||
if(after.dev!==before.dev||after.ino!==before.ino||after.size!==before.size)throw new Error("manual distribution module changed while binding");
|
||||
if(createHash("sha256").update(bytes).digest("hex")!==file.sha256)throw new Error("manual distribution module bytes changed");
|
||||
distBytes.set(rel,bytes);
|
||||
}finally{closeSync(fd);}
|
||||
}
|
||||
if(!distBytes.has("server.js")||createHash("sha256").update(entrySource).digest("hex")!==manifest.files["server.js"].sha256)throw new Error("manual entrypoint manifest identity refused");
|
||||
const entryUrl=pathToFileURL(entryPath).href,distPrefix=distRoot+sep;
|
||||
registerHooks({load(url,context,nextLoad){if(url===entryUrl)return{format:"module",shortCircuit:true,source:entrySource};let pathname;try{pathname=fileURLToPath(url);}catch{return nextLoad(url,context);}if(pathname.startsWith(distPrefix)){const rel=pathname.slice(distPrefix.length);const bytes=distBytes.get(rel);if(!bytes)throw new Error("manual distribution module refused");return{format:rel.endsWith(".json")?"json":"module",shortCircuit:true,source:bytes};}return nextLoad(url,context);}});
|
||||
let state="STARTING",stopping=false,ownedListener,listenGeneration=0;
|
||||
const listenerIdentity=()=>{const address=ownedListener?.listening?ownedListener.address():undefined;return{listening:Boolean(ownedListener?.listening&&address&&address.address===HOST&&address.port===HTTP_PORT),host:address?.address,port:address?.port,generation:listenGeneration};};
|
||||
const originalListen=net.Server.prototype.listen;net.Server.prototype.listen=function(...args){const candidate=this;candidate.once("listening",()=>{const address=candidate.address();if(address&&address.address===HOST&&address.port===HTTP_PORT){ownedListener=candidate;listenGeneration++;}});candidate.on("close",()=>{if(ownedListener===candidate){ownedListener=undefined;if(state==="READY")state="LISTENER_CLOSED";}});return originalListen.apply(candidate,args);};
|
||||
const identity=()=>({status:state,pid:process.pid,nonce,controlNonce,root,control:{host:HOST,port:PORT},listener:listenerIdentity()});
|
||||
const control=net.createServer(socket=>{let bytes="";socket.setEncoding("utf8");socket.on("data",chunk=>{bytes+=chunk;if(bytes.length>2048)socket.destroy();});socket.on("end",()=>{let request;try{request=JSON.parse(bytes);}catch{socket.end();return;}if(request?.nonce!==controlNonce){socket.end();return;}if(request.action==="status"){socket.end(JSON.stringify(identity())+"\\n");return;}if(request.action==="ready"&&!stopping&&listenerIdentity().listening){state="READY";clearTimeout(watchdog);socket.end(JSON.stringify(identity())+"\\n");return;}if(request.action==="stop"&&!stopping){stopping=true;state="STOPPING";clearTimeout(watchdog);socket.end(JSON.stringify(identity())+"\\n",()=>{control.close();if(ownedListener?.listening)ownedListener.close(()=>process.exit(0));else setImmediate(()=>process.exit(0));});return;}socket.end(JSON.stringify(identity())+"\\n");});});
|
||||
await new Promise((resolve,reject)=>{control.once("error",reject);control.listen({host:HOST,port:PORT,exclusive:true},resolve);});
|
||||
const watchdog=setTimeout(()=>{if(state!=="STARTING")return;console.error("manual backend readiness watchdog expired");control.close(()=>process.exit(1));setTimeout(()=>process.exit(1),100).unref();},8000);
|
||||
`;
|
||||
const PRELOAD=`data:text/javascript;base64,${Buffer.from(PRELOAD_SOURCE,"utf8").toString("base64")}`;
|
||||
async function controlRequest(control,payload){if(control?.host!==HOST||!Number.isSafeInteger(control?.port)||control.port<1||control.port>65535)throw new Error("backend control identity mismatch");return await new Promise((resolvePromise,reject)=>{const socket=net.createConnection({host:control.host,port:control.port}),timer=setTimeout(()=>socket.destroy(new Error("backend control timeout")),2000);let bytes="";socket.setEncoding("utf8");socket.on("connect",()=>socket.end(JSON.stringify(payload)));socket.on("data",chunk=>{bytes+=chunk;if(bytes.length>2048)socket.destroy(new Error("backend control response too large"));});socket.on("error",reject);socket.on("close",()=>{clearTimeout(timer);let value;try{value=JSON.parse(bytes);}catch{return reject(new Error("backend control response is malformed"));}resolvePromise(value);});});}
|
||||
|
||||
function ownedValue(repo,root,nonce,{backendLog=null,entrypoint,distManifest=null,stage="PREPARING",createdAt=new Date().toISOString()}={}){return{schemaVersion:1,kind:"p1-manual-acceptance",nonce,repositoryRoot:repo,root,status:"PENDING",stage,createdAt,listener:{host:HOST,port:PORT,state:"stopped"},backendLog,entrypoint,distManifest,resources:[root,{kind:"fastify",host:HOST,port:PORT}]};}
|
||||
function validEntrypoint(value,repo){return value?.path===join(repo,"backend/dist/server.js")&&Number.isSafeInteger(value.dev)&&Number.isSafeInteger(value.ino)&&Number.isSafeInteger(value.size)&&value.size>0&&HEX64.test(value.sha256??"");}
|
||||
async function readBoundEntrypoint(repo){
|
||||
if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("production entrypoint no-follow protection is unavailable");const path=join(repo,"backend/dist/server.js");let handle;
|
||||
try{handle=await open(path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat(),pathEntry=await lstat(path);if(!before.isFile()||before.nlink!==1||pathEntry.isSymbolicLink()||!pathEntry.isFile()||!sameEntry(before,pathEntry))throw new Error("production server identity is unsafe");if(before.size<1||before.size>33554432)throw new Error("production entrypoint is unbounded");const bytes=Buffer.alloc(before.size);let offset=0;while(offset<bytes.length){const{bytesRead}=await handle.read(bytes,offset,bytes.length-offset,offset);if(bytesRead<1)throw new Error("production entrypoint changed while binding");offset+=bytesRead;}const after=await handle.stat();if(!sameEntry(before,after)||before.size!==bytes.length||after.size!==before.size)throw new Error("production entrypoint changed while binding");return{handle,identity:{path,dev:before.dev,ino:before.ino,size:before.size,sha256:createHash("sha256").update(bytes).digest("hex")},bytes};}catch(error){if(handle)await handle.close().catch(()=>{});throw error;}
|
||||
}
|
||||
async function requireEntrypointPathIdentity(entrypoint){const entry=await lstat(entrypoint.path);if(!entry.isFile()||entry.isSymbolicLink()||entry.nlink!==1||entry.dev!==entrypoint.dev||entry.ino!==entrypoint.ino||entry.size!==entrypoint.size)throw new Error("production entrypoint identity changed");const bytes=await readFile(entrypoint.path);if(bytes.length!==entrypoint.size||createHash("sha256").update(bytes).digest("hex")!==entrypoint.sha256)throw new Error("production entrypoint bytes changed");return entry;}
|
||||
function validDistManifest(value,root){return value?.path===join(root,"installation","runtime","backend-dist.manifest.json")&&Number.isSafeInteger(value.dev)&&Number.isSafeInteger(value.ino)&&Number.isSafeInteger(value.size)&&value.size>0&&HEX64.test(value.sha256??"");}
|
||||
function parseDistManifest(bytes,distRoot){let value;try{value=JSON.parse(bytes.toString("utf8"));}catch{throw new Error("production distribution manifest is malformed");}const files=value?.files;if(value?.schemaVersion!==1||value.kind!=="p1-manual-dist-manifest"||value.root!==distRoot||!files||typeof files!=="object"||Array.isArray(files))throw new Error("production distribution manifest is malformed");const entries=Object.entries(files);if(entries.length<1||entries.length>20000)throw new Error("production distribution manifest is malformed");for(const[rel,file]of entries){if(!/^[^./\\][^/\\]*(?:\/[^./\\][^/\\]*)*$/.test(rel)||!Number.isSafeInteger(file?.size)||file.size<1||file.size>33554432||!HEX64.test(file?.sha256??"")||!Number.isSafeInteger(file?.dev)||!Number.isSafeInteger(file?.ino))throw new Error("production distribution manifest is malformed");}return{value,files};}
|
||||
async function buildDistManifest(repo){const dist=join(repo,"backend","dist"),files={};let count=0,total=0;async function walk(dir){for(const entry of await readdir(dir,{withFileTypes:true})){const path=join(dir,entry.name);if(entry.isSymbolicLink())throw new Error("production distribution contains a symlink");if(entry.isDirectory()){await walk(path);continue;}if(!entry.isFile())throw new Error("production distribution contains a nonregular entry");if(++count>20000)throw new Error("production distribution is unbounded");const rel=relative(dist,path).split(sep).join("/");let handle;try{handle=await open(path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat();if(!before.isFile()||before.nlink!==1||before.size<1||before.size>33554432)throw new Error("production distribution module is unsafe");total+=before.size;if(total>536870912)throw new Error("production distribution is unbounded");const bytes=Buffer.alloc(before.size);let offset=0;while(offset<bytes.length){const{bytesRead}=await handle.read(bytes,offset,bytes.length-offset,offset);if(bytesRead<1)throw new Error("production distribution module changed while binding");offset+=bytesRead;}const after=await handle.stat();if(!sameEntry(before,after)||after.size!==before.size)throw new Error("production distribution module changed while binding");files[rel]={size:before.size,sha256:createHash("sha256").update(bytes).digest("hex"),dev:before.dev,ino:before.ino};}finally{if(handle)await handle.close().catch(()=>{});}}}await walk(dist);return{schemaVersion:1,kind:"p1-manual-dist-manifest",root:dist,files};}
|
||||
async function readBoundDistManifest(repo,owned){
|
||||
if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("production distribution manifest no-follow protection is unavailable");const distManifest=owned.distManifest;let handle;
|
||||
try{handle=await open(distManifest.path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat(),pathEntry=await lstat(distManifest.path);if(!before.isFile()||before.nlink!==1||pathEntry.isSymbolicLink()||!pathEntry.isFile()||!sameEntry(before,pathEntry)||before.dev!==distManifest.dev||before.ino!==distManifest.ino||before.size!==distManifest.size)throw new Error("production distribution manifest identity changed");if(before.size<1||before.size>8388608)throw new Error("production distribution manifest is unbounded");const bytes=Buffer.alloc(before.size);let offset=0;while(offset<bytes.length){const{bytesRead}=await handle.read(bytes,offset,bytes.length-offset,offset);if(bytesRead<1)throw new Error("production distribution manifest changed while binding");offset+=bytesRead;}const after=await handle.stat();if(!sameEntry(before,after)||before.size!==bytes.length||after.size!==before.size)throw new Error("production distribution manifest changed while binding");if(createHash("sha256").update(bytes).digest("hex")!==distManifest.sha256)throw new Error("production distribution manifest bytes changed");const{files}=parseDistManifest(bytes,join(repo,"backend","dist"));if(files["server.js"]?.sha256!==owned.entrypoint.sha256)throw new Error("production distribution manifest does not bind the entrypoint");return{handle,files};}catch(error){if(handle)await handle.close().catch(()=>{});throw error;}
|
||||
}
|
||||
async function validateDistFiles(repo,files){const dist=join(repo,"backend","dist");for(const[rel,file]of Object.entries(files)){const path=join(dist,...rel.split("/"));let handle;try{handle=await open(path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat(),pathEntry=await lstat(path);if(!before.isFile()||before.nlink!==1||pathEntry.isSymbolicLink()||!pathEntry.isFile()||!sameEntry(before,pathEntry)||before.dev!==file.dev||before.ino!==file.ino||before.size!==file.size)throw new Error("production distribution module identity changed");const bytes=Buffer.alloc(before.size);let offset=0;while(offset<bytes.length){const{bytesRead}=await handle.read(bytes,offset,bytes.length-offset,offset);if(bytesRead<1)throw new Error("production distribution module changed while binding");offset+=bytesRead;}const after=await handle.stat();if(!sameEntry(before,after)||after.size!==before.size)throw new Error("production distribution module changed while binding");if(createHash("sha256").update(bytes).digest("hex")!==file.sha256)throw new Error("production distribution module bytes changed");}finally{if(handle)await handle.close().catch(()=>{});}}}
|
||||
|
||||
export async function readManualOwnership({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),root=fixedManualRoot(repo);noSymlinkExisting(repo,root);let rootEntry,ownershipEntry;try{rootEntry=await lstat(root);ownershipEntry=await lstat(join(root,"ownership.json"));}catch{throw new Error("manual ownership is missing");}if(!rootEntry.isDirectory()||rootEntry.isSymbolicLink()||await realpath(root)!==root||!ownershipEntry.isFile()||ownershipEntry.isSymbolicLink())throw new Error("manual ownership is unsafe");let value;try{value=JSON.parse(await readFile(join(root,"ownership.json"),"utf8"));}catch{throw new Error("manual ownership is malformed");}const baseValid=value.schemaVersion===1&&value.kind==="p1-manual-acceptance"&&HEX64.test(value.nonce??"")&&value.repositoryRoot===repo&&value.root===root&&value.status==="PENDING"&&["PREPARING","READY"].includes(value.stage)&&value.listener?.host===HOST&&value.listener?.port===PORT&&value.listener?.state==="stopped"&&typeof value.createdAt==="string"&&validEntrypoint(value.entrypoint,repo)&&validDistManifest(value.distManifest,root)&&JSON.stringify(value.resources)===JSON.stringify([root,{kind:"fastify",host:HOST,port:PORT}]);const readyLog=value.backendLog?.path===join(root,"logs/backend.log")&&Number.isSafeInteger(value.backendLog?.dev)&&Number.isSafeInteger(value.backendLog?.ino);if(!baseValid||(value.stage==="READY"?!readyLog:value.backendLog!==null))throw new Error("manual ownership identity mismatch");return value;}
|
||||
async function run(executable,argv,options={}){return await exec(executable,argv,{...options,maxBuffer:2*1024*1024,encoding:"utf8"});}
|
||||
function descriptor(id,source){return{workspace:{schema_version:3,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},semantic_index:{vector_store:{engine:"qdrant",collection:id,dimensions:1024,distance:"cosine"},embedding:{provider:"ollama_internal",model:"qwen3-embedding:0.6b",dimensions:1024}},llm_policy:{allowed:["zai/glm-5.2"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
|
||||
function descriptors(){return[descriptor("p1-filesystem",{type:"filesystem",uri:"workspace-content/p1-filesystem/evidence",patterns:["**/*.md"],max_bytes:10485760}),descriptor("p1-http",{type:"http",uris:["https://evidence.example.test/guide.md"],authentication:"signed_urls_file",connect_timeout_ms:1250,read_timeout_ms:30001,max_bytes:12345,max_redirects:2,allow_private_hosts:false,max_cache_bytes:67890}),descriptor("p1-s3",{type:"s3",uri:"s3://p1-evidence/published/",endpoint_url:"https://s3.example.test/",region:"eu-west-1",credentials:"static_files",trusted_endpoint:true,allow_private_endpoint:false,allow_insecure_endpoint:false,max_bytes:12345,max_objects:33,max_pages:4,page_size:5})];}
|
||||
function quote(value){return `'${String(value).replaceAll("'",`'"'"'`)}'`;}
|
||||
async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof","python3"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}}
|
||||
async function initializeGit(root){await run("git",["init","--bare","--initial-branch=main",join(root,"remote.git")],{cwd:root});await run("git",["clone",join(root,"remote.git"),join(root,"author")],{cwd:root});for(const [key,value]of [["user.name","P1 Manual Curator"],["user.email","p1-manual@example.invalid"]])await run("git",["config",key,value],{cwd:join(root,"author")});const evidence=join(root,"author","workspace-content","p1-filesystem","evidence");await mkdir(join(evidence,"domain"),{recursive:true});await writeFile(join(evidence,"guide.md"),"# P1 manually curated Evidence\n");await writeFile(join(evidence,"domain","table.md"),"# P1 curated table\n");await run("git",["add","workspace-content"],{cwd:join(root,"author")});await run("git",["commit","-m","Bootstrap P1 manual Evidence"],{cwd:join(root,"author")});await run("git",["push","origin","main"],{cwd:join(root,"author")});}
|
||||
function requestFixtures(items){const result={"status.json":{method:"GET",path:"/workspace-registry/status"},"pull.json":{method:"POST",path:"/workspace-registry/pull"}};for(const workspace of items){const id=workspace.workspace.id;result[`validate-${id}.json`]={workspace};result[`publish-${id}.json`]={action:"create",workspace};result[`read-${id}.json`]={method:"GET",path:`/workspaces/${id}`};result[`export-${id}.json`]={method:"GET",path:`/workspaces/${id}/export`};}Object.assign(result,{"invalid-absolute.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"/etc"}}}},"invalid-traversal.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-filesystem/evidence/../../p1-s3/evidence"}}}},"invalid-cross-workspace.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-s3/evidence"}}}},"invalid-protocol.json":{workspace:{...items[1],evidence:{...items[1].evidence,source:{...items[1].evidence.source,uris:["file:///etc/passwd"]}}}},"invalid-credential.json":{workspace:{...items[2],evidence:{...items[2].evidence,source:{...items[2].evidence.source,access_key:"CANARY-MUST-BE-REJECTED"}}}}});return result;}
|
||||
function curlGet(url,output){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;}
|
||||
function curlPost(url,output,body){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(body)} --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;}
|
||||
function curlPostEmpty(url,output){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;}
|
||||
function publishCurl(root,id,previousResponse){const descriptor=join(root,"fixtures/descriptors",`${id}.json`),body=join(root,"requests",`publish-${id}.concrete.json`),response=join(root,"responses",`publish-${id}.json`);return `#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
node --input-type=module - ${quote(previousResponse)} ${quote(descriptor)} ${quote(body)} <<'NODE'
|
||||
import { open, readFile, rename, stat } from "node:fs/promises";import { basename, dirname, join } from "node:path";import { randomBytes } from "node:crypto";
|
||||
const [priorPath,descriptorPath,output]=process.argv.slice(2);const bounded=async(path)=>{let s;try{s=await stat(path);}catch{throw Error("required saved response is missing");}if(!s.isFile()||s.size<2||s.size>1048576)throw Error("saved response is unbounded");let value;try{value=JSON.parse(await readFile(path,"utf8"));}catch{throw Error("saved response is malformed JSON");}return value;};
|
||||
const prior=await bounded(priorPath),workspace=await bounded(descriptorPath);const base=prior.head??prior.revision?.commit;if(!/^[0-9a-f]{40}$/.test(base??""))throw Error("saved response has no valid current base commit");const bytes=JSON.stringify({action:"create",workspace,baseCommit:base},null,2)+"\\n",tmp=join(dirname(output),"."+basename(output)+"."+randomBytes(8).toString("hex")+".tmp");const h=await open(tmp,"wx",0o600);try{await h.writeFile(bytes);await h.sync();}finally{await h.close();}await rename(tmp,output);
|
||||
NODE
|
||||
curl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(body)} --output ${quote(response)} --write-out 'HTTP %{http_code}\n' 'http://127.0.0.1:8791/workspaces/publish'
|
||||
`;}
|
||||
function httpCommands(root){const base="http://127.0.0.1:8791",entries=[];entries.push(["http-01-status.sh",curlGet(`${base}/workspace-registry/status`,join(root,"responses/status.json"))]);let n=2;for(const id of ["p1-filesystem","p1-http","p1-s3"])entries.push([`http-0${n++}-validate-${id}.sh`,curlPost(`${base}/workspaces/validate`,join(root,"responses",`validate-${id}.json`),join(root,"requests",`validate-${id}.json`))]);let prior=join(root,"responses/status.json");for(const id of ["p1-filesystem","p1-http","p1-s3"]){entries.push([`http-0${n++}-publish-${id}.sh`,publishCurl(root,id,prior)]);prior=join(root,"responses",`publish-${id}.json`);}entries.push([`http-0${n++}-pull.sh`,curlPostEmpty(`${base}/workspace-registry/pull`,join(root,"responses/pull.json"))]);for(const id of ["p1-filesystem","p1-http","p1-s3"])entries.push([`http-${String(n++).padStart(2,"0")}-read-${id}.sh`,curlGet(`${base}/workspaces/${id}`,join(root,"responses",`read-${id}.json`))]);for(const id of ["p1-filesystem","p1-http","p1-s3"])entries.push([`http-${String(n++).padStart(2,"0")}-export-${id}.sh`,curlGet(`${base}/workspaces/${id}/export`,join(root,"exports/raw",`${id}.zip`))]);for(const kind of ["absolute","traversal","cross-workspace","protocol","credential"])entries.push([`http-${String(n++).padStart(2,"0")}-invalid-${kind}.sh`,curlPost(`${base}/workspaces/validate`,join(root,"responses",`invalid-${kind}.json`),join(root,"requests",`invalid-${kind}.json`))]);return entries;}
|
||||
function renderCommand(repo,root,n){const output=join(root,"rendered",`runtime-${n}.yaml`),response=join(root,"responses","read-p1-filesystem.json"),published=join(root,"responses","pull.json"),snapshots=join(root,"installation","registry","snapshots"),checkout=join(root,"installation","registry","repo");return `#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
repo=${quote(repo)}
|
||||
root=${quote(root)}
|
||||
set -a
|
||||
. ${quote(join(root,"installation","bindings.env"))}
|
||||
set +a
|
||||
node --input-type=module - "$root" ${quote(response)} ${quote(published)} ${quote(snapshots)} ${quote(checkout)} ${quote(output)} "$repo/backend/scripts/p1-render-snapshot.mjs" <<'NODE'
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, realpath, stat } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
const [root,readPath,publishPath,snapshots,checkout,output,renderer]=process.argv.slice(2);
|
||||
const HEX40=/^[0-9a-f]{40}$/,HEX64=/^[0-9a-f]{64}$/,REVISION_KEYS=["blob","commit","id","snapshotPath"];
|
||||
const bounded=async(path,label="saved response")=>{let s;try{s=await stat(path);}catch{throw new Error(label+" is missing or unbounded");}if(!s.isFile()||s.size<2||s.size>1048576)throw new Error(label+" is missing or unbounded");let v;try{v=JSON.parse(await readFile(path,"utf8"));}catch{throw new Error(label+" is malformed JSON");}return v;};
|
||||
const boundedBytes=async(path)=>{let s;try{s=await stat(path);}catch{throw new Error("saved snapshot is missing or unbounded");}if(!s.isFile()||s.size<2||s.size>1048576)throw new Error("saved snapshot is missing or unbounded");return await readFile(path);};
|
||||
const read=await bounded(readPath),published=await bounded(publishPath);const revision=read?.revision,commit=revision?.commit,snapshot=revision?.snapshotPath,publishedCommit=published?.head??published?.revision?.commit;
|
||||
if(!HEX40.test(commit??"")||commit!==publishedCommit)throw new Error("saved read/publish revisions differ");
|
||||
if(typeof snapshot!=="string"||!isAbsolute(snapshot))throw new Error("snapshot path is not absolute");const canonical=await realpath(snapshot);const rel=relative(snapshots,canonical);if(rel.startsWith("..")||isAbsolute(rel)||dirname(canonical)!==resolve(snapshots,commit))throw new Error("snapshot escapes owned commit root");
|
||||
const id=basename(canonical).slice(0,-".yaml".length);if(!/^[a-z][a-z0-9-]{2,62}$/.test(id))throw new Error("snapshot workspace identity is invalid");
|
||||
const git=spawnSync("git",["-C",checkout,"rev-parse","HEAD"],{encoding:"utf8"});if(git.status!==0||git.stdout.trim()!==commit)throw new Error("saved revision differs from installed Git commit");
|
||||
const manifest=await bounded(join(snapshots,commit,"snapshot.json"),"snapshot manifest");const files=manifest?.files,revisions=manifest?.revisions;
|
||||
if(manifest?.head!==commit||!files||typeof files!=="object"||Array.isArray(files))throw new Error("snapshot manifest identity is invalid");
|
||||
const expected=files[id+".yaml"];if(!HEX64.test(expected??""))throw new Error("snapshot manifest digest is invalid");
|
||||
const snapshotBytes=await boundedBytes(canonical);if(createHash("sha256").update(snapshotBytes).digest("hex")!==expected)throw new Error("snapshot bytes differ from manifest digest");
|
||||
const entry=Array.isArray(revisions)?revisions.find(candidate=>candidate?.id===id):undefined;
|
||||
const exactEntry=entry&&typeof entry==="object"&&!Array.isArray(entry)&&Object.keys(entry).sort().every((key,index)=>key===REVISION_KEYS[index])&&Object.keys(entry).length===REVISION_KEYS.length;
|
||||
if(!exactEntry||entry.id!==id||entry.commit!==commit||typeof entry.blob!=="string"||!HEX40.test(entry.blob)||entry.snapshotPath!==canonical)throw new Error("snapshot manifest revision is invalid");
|
||||
if(!HEX40.test(revision?.blob??"")||revision.blob!==entry.blob)throw new Error("saved revision blob differs from snapshot manifest");
|
||||
const blobCheck=spawnSync("git",["-C",checkout,"rev-parse",commit+":workspaces/"+id+".yaml"],{encoding:"utf8"});
|
||||
if(blobCheck.status!==0||blobCheck.stdout.trim()!==entry.blob)throw new Error("snapshot blob differs from installed Git commit");
|
||||
const hashObject=spawnSync("git",["hash-object","--stdin"],{input:snapshotBytes,encoding:"utf8"});
|
||||
if(hashObject.status!==0||hashObject.stdout.trim()!==entry.blob)throw new Error("snapshot bytes differ from Git blob");
|
||||
const child=spawnSync(process.execPath,[renderer,"--ownership",resolve(root,"ownership.json"),"--snapshot",canonical,"--output",output,"--snapshot-sha256",expected],{stdio:"inherit",env:process.env});if(child.status!==0)process.exit(child.status??1);
|
||||
NODE
|
||||
`;}
|
||||
function guide(repo,root){const base=`http://${HOST}:${PORT}`;return `# P1 manual configuration walkthrough
|
||||
|
||||
Status: **PENDING**. The reviewer, not this helper, performs and judges every step. Never inspect raw secret-file contents. Every lifecycle action uses the stable repository-root \`.p1-manual-acceptance.lifecycle.lock\`; successful prepare has advanced its ownership-first recovery record from \`PREPARING\` to \`READY\`.
|
||||
|
||||
1. Inspect \`${root}/ownership.json\`, including the bound production entrypoint identity and the complete \`backend/dist\` module manifest identity, the pre-publication Evidence tree under \`author/workspace-content/p1-filesystem/evidence\`, descriptor fixtures, and binding **paths and modes** in \`installation/bindings.env\`.
|
||||
2. Run \`${repo}/scripts/p1-manual-acceptance.sh serve\`; verify one production Node PID owns both \`${HOST}:${PORT}\` and its authenticated \`${HOST}:${CONTROL_PORT}\` control listener (for example, use \`lsof -nP -iTCP:${PORT} -sTCP:LISTEN\` and repeat for port ${CONTROL_PORT}). Serve executes the ownership-bound production entrypoint and complete verified \`backend/dist\` module graph from opened no-follow descriptors and publishes \`RUNNING\` only after the same authenticated child acknowledges its owned HTTP listener and passes bounded health checks.
|
||||
3. Personally run each concrete \`commands/http-01-*.sh\` through \`commands/http-14-*.sh\` script, one at a time in numeric order: real curl status → three validates → three sequential publishes → pull → three reads → three exports against \`${base}\`. Each script saves the exact JSON response under \`responses/\` or ZIP bytes under \`exports/raw/\`; each publish derives its current base from the preceding bounded saved response. Do not advance on a non-2xx response.
|
||||
4. Only after publish, run \`commands/git-inspect.sh <published-commit>\`: inspect \`git log\`, \`git ls-tree\`, \`git show <published-commit>:workspaces/<id>.yaml\`, and \`git show <published-commit>:workspace-content/<id>/evidence/...\` at that same commit.
|
||||
5. Inspect generated \`workspace-docs\`, the immutable commit-addressed descriptor snapshot, and its \`snapshot.json\` manifest.
|
||||
6. Run \`commands/extract-export.sh exports/raw/p1-filesystem.zip exports/extracted/p1-filesystem p1-filesystem\`, then the equivalent exact commands for \`p1-http\` and \`p1-s3\`; verify each manifest and descriptor identity, hashes, and absence of Evidence bytes and secret/canary material.
|
||||
7. After saving \`responses/read-p1-filesystem.json\` and the final API/Git head in \`responses/pull.json\`, run \`commands/render-1.sh\`, \`commands/render-2.sh\`, then \`commands/diff-rendered.sh\`. The render commands bind the snapshot bytes to the commit\'s \`snapshot.json\` digest and Git blob identity; the renderer revalidates that digest and renders only the verified bytes through one opened no-follow \`rendered\` directory identity, refusing an ancestor swap.
|
||||
8. Inspect runtime identity, absolute reserved filesystem root, Evidence limits, and policy in the rendered YAML; do not inspect secret contents.
|
||||
9. Personally execute \`${repo}/harness/.venv/bin/tht config check -c ${root}/rendered/runtime-1.yaml\` and the same command for \`runtime-2.yaml\` (or run \`commands/config-check.sh\`).
|
||||
10. Personally run \`commands/http-15-*.sh\` through \`commands/http-19-*.sh\` to submit the invalid absolute, Evidence-URI traversal, cross-workspace, protocol, and credential validation requests; verify safe rejection, no Git/snapshot mutation, and no rejected canary outside the request fixture.
|
||||
11. Run \`commands/secret-scan.sh\`; it excludes only the direct \`fixture-secrets\` payload directory, scans bounded filesystem content and name/path bytes, discovers every bounded arbitrary \`.git\` repository plus the owned bare remote, and checks loose-ref names plus raw bounded bytes from every blob, commit, tree, and tag object, including unreachable objects. Findings and operational errors redact secret-bearing paths and values.
|
||||
12. Run \`commands/absence-check.sh\`; confirm no file or directory represents preprocessing, Evidence materialization (including \`artifacts/evidence\`), embedding, Qdrant, ACTIVE, or retention state.
|
||||
13. Run \`${repo}/scripts/p1-manual-acceptance.sh stop\`; confirm \`backend.pid\` and both listeners on ports ${PORT} and ${CONTROL_PORT} are gone.
|
||||
14. Create \`${root}/VERDICT.md\` yourself with reviewer, UTC time, every checklist result, observations, and exactly either \`manual acceptance: PASS\` or \`manual acceptance: FAIL\`.
|
||||
|
||||
Preserve a failed lab by stopping it and leaving the owned root in place. Only \`cleanup\` removes this exact stopped lab.
|
||||
`;}
|
||||
function extractCommand(repo,root){return `#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
zip=\${1:?zip required}; out=\${2:?new output required}; expected=\${3:?expected workspace id required}
|
||||
python3 - "$zip" "$out" "$expected" ${quote(root)} ${quote(join(repo,"backend/package.json"))} <<'PY'
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
zip_path, output_path, expected, root, package_json = sys.argv[1:]
|
||||
allowed = {"p1-filesystem", "p1-http", "p1-s3"}
|
||||
required = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"]
|
||||
base = os.path.join(root, "exports", "extracted")
|
||||
base_fd = None
|
||||
archive_fd = None
|
||||
stage_fd = None
|
||||
archive_stage = None
|
||||
extract_stage = None
|
||||
published = False
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def exact(value, keys):
|
||||
return isinstance(value, dict) and set(value) == set(keys)
|
||||
|
||||
|
||||
def write_all(fd, data):
|
||||
view = memoryview(data)
|
||||
while view:
|
||||
written = os.write(fd, view)
|
||||
if written <= 0:
|
||||
fail("anchored extraction write failed")
|
||||
view = view[written:]
|
||||
|
||||
|
||||
def read_exact_fd(fd, expected_size, limit, label):
|
||||
if expected_size < 1 or expected_size > limit:
|
||||
fail(label + " is unbounded")
|
||||
chunks = []
|
||||
remaining = expected_size
|
||||
while remaining:
|
||||
chunk = os.read(fd, min(1024 * 1024, remaining))
|
||||
if not chunk:
|
||||
fail(label + " changed while staging")
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
if os.read(fd, 1):
|
||||
fail(label + " changed while staging")
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def require_base_identity():
|
||||
try:
|
||||
current = os.stat(base, follow_symlinks=False)
|
||||
except OSError:
|
||||
fail("owned extraction root identity changed")
|
||||
if (not stat.S_ISDIR(current.st_mode) or current.st_dev != base_identity.st_dev
|
||||
or current.st_ino != base_identity.st_ino or os.path.realpath(base) != base):
|
||||
fail("owned extraction root identity changed")
|
||||
|
||||
|
||||
def remove_anchored_directory(name):
|
||||
child_fd = None
|
||||
try:
|
||||
child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd)
|
||||
for entry in os.listdir(child_fd):
|
||||
if entry not in required:
|
||||
fail("anchored extraction cleanup found an unexpected entry")
|
||||
os.unlink(entry, dir_fd=child_fd)
|
||||
os.fsync(child_fd)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
finally:
|
||||
if child_fd is not None:
|
||||
os.close(child_fd)
|
||||
os.rmdir(name, dir_fd=base_fd)
|
||||
os.fsync(base_fd)
|
||||
|
||||
|
||||
try:
|
||||
for flag in ("O_DIRECTORY", "O_NOFOLLOW"):
|
||||
if not hasattr(os, flag):
|
||||
fail("anchored extraction is unavailable on this platform")
|
||||
if expected not in allowed:
|
||||
fail("expected workspace identity is invalid")
|
||||
if os.path.realpath(root) != root or os.path.dirname(output_path) != base:
|
||||
fail("unsafe owned extraction root or output path")
|
||||
output_name = os.path.basename(output_path)
|
||||
if not output_name or output_name.startswith(".") or os.sep in output_name:
|
||||
fail("unsafe output path")
|
||||
|
||||
base_fd = os.open(base, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
base_identity = os.fstat(base_fd)
|
||||
if not stat.S_ISDIR(base_identity.st_mode):
|
||||
fail("unsafe owned extraction root")
|
||||
require_base_identity()
|
||||
try:
|
||||
os.stat(output_name, dir_fd=base_fd, follow_symlinks=False)
|
||||
fail("unsafe output path")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
# Open the caller's source exactly once, then consume only an owned staged copy.
|
||||
source_fd = os.open(zip_path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
try:
|
||||
source_identity = os.fstat(source_fd)
|
||||
if not stat.S_ISREG(source_identity.st_mode):
|
||||
fail("source ZIP is unsafe")
|
||||
source_bytes = read_exact_fd(source_fd, source_identity.st_size, 33554432, "source ZIP")
|
||||
source_after = os.fstat(source_fd)
|
||||
if (source_after.st_dev, source_after.st_ino, source_after.st_size) != (source_identity.st_dev, source_identity.st_ino, source_identity.st_size):
|
||||
fail("source ZIP changed during staging")
|
||||
finally:
|
||||
os.close(source_fd)
|
||||
archive_sha = hashlib.sha256(source_bytes).digest()
|
||||
|
||||
archive_stage = ".zip-stage-" + secrets.token_hex(16) + ".zip"
|
||||
archive_fd = os.open(archive_stage, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=base_fd)
|
||||
write_all(archive_fd, source_bytes)
|
||||
os.fsync(archive_fd)
|
||||
del source_bytes
|
||||
os.lseek(archive_fd, 0, os.SEEK_SET)
|
||||
staged_bytes = read_exact_fd(archive_fd, os.fstat(archive_fd).st_size, 33554432, "staged archive")
|
||||
if hashlib.sha256(staged_bytes).digest() != archive_sha:
|
||||
fail("staged archive SHA mismatch")
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(staged_bytes), "r") as archive:
|
||||
infos = archive.infolist()
|
||||
names = [entry.filename for entry in infos]
|
||||
if len(names) != 4 or len(set(names)) != 4 or set(names) != set(required):
|
||||
fail("unsafe-zip entries")
|
||||
for entry in infos:
|
||||
mode = (entry.external_attr >> 16) & 0xFFFF
|
||||
if not stat.S_ISREG(mode) or entry.flag_bits & 1:
|
||||
fail("ZIP contains a symlink or nonregular entry")
|
||||
if entry.file_size < 1 or entry.file_size > 10485760:
|
||||
fail("extracted file is unsafe")
|
||||
payloads = {name: archive.read(name) for name in required}
|
||||
if any(len(payloads[entry.filename]) != entry.file_size for entry in infos):
|
||||
fail("extracted file size mismatch")
|
||||
|
||||
# Exercise the documented unzip prerequisite against the exact staged descriptor, not a path.
|
||||
listing = subprocess.run(
|
||||
["unzip", "-Z1", "/dev/fd/" + str(archive_fd)], pass_fds=(archive_fd,),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20, check=False,
|
||||
)
|
||||
if listing.returncode != 0 or listing.stdout.decode("utf8", "strict").splitlines() != names:
|
||||
fail("unsafe-zip entries")
|
||||
os.lseek(archive_fd, 0, os.SEEK_SET)
|
||||
revalidated = read_exact_fd(archive_fd, len(staged_bytes), 33554432, "staged archive")
|
||||
if hashlib.sha256(revalidated).digest() != archive_sha or revalidated != staged_bytes:
|
||||
fail("staged archive SHA mismatch")
|
||||
require_base_identity()
|
||||
|
||||
randomized = re.compile(br"(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}")
|
||||
fixed = b"-".join([b"CANARY", b"MUST", b"BE", b"REJECTED"])
|
||||
for data in payloads.values():
|
||||
if b"P1 manually curated Evidence" in data or b"P1 curated table" in data or randomized.search(data) or fixed in data:
|
||||
fail("export contains Evidence or secret canary bytes")
|
||||
|
||||
try:
|
||||
manifest = json.loads(payloads["manifest.json"].decode("utf8"))
|
||||
except Exception:
|
||||
fail("export manifest schema mismatch")
|
||||
hashed = required[1:]
|
||||
files = manifest.get("files") if isinstance(manifest, dict) else None
|
||||
if (not exact(manifest, ["schema_version", "workspace_id", "files"])
|
||||
or manifest.get("schema_version") != 1 or manifest.get("workspace_id") != expected
|
||||
or not exact(files, hashed)
|
||||
or any(not isinstance(files[name], str) or not re.fullmatch(r"[0-9a-f]{64}", files[name]) for name in hashed)):
|
||||
fail("export manifest workspace identity or schema mismatch")
|
||||
for name in hashed:
|
||||
if hashlib.sha256(payloads[name]).hexdigest() != files[name]:
|
||||
fail("manifest hash mismatch")
|
||||
|
||||
yaml_helper = 'const fs=require("node:fs"),{createRequire}=require("node:module");try{const YAML=createRequire(process.argv[1])("yaml"),v=YAML.parse(fs.readFileSync(0,"utf8"));process.stdout.write(JSON.stringify(v?.workspace?.id??null));}catch{process.exit(2)}'
|
||||
parsed = subprocess.run(["node", "-e", yaml_helper, package_json], input=payloads["workspace.yaml"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10, check=False)
|
||||
try:
|
||||
descriptor_id = json.loads(parsed.stdout.decode("utf8")) if parsed.returncode == 0 else None
|
||||
except Exception:
|
||||
descriptor_id = None
|
||||
if descriptor_id != expected:
|
||||
fail("export descriptor workspace identity mismatch")
|
||||
|
||||
extract_stage = ".extract-stage-" + secrets.token_hex(16)
|
||||
os.mkdir(extract_stage, 0o700, dir_fd=base_fd)
|
||||
stage_fd = os.open(extract_stage, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd)
|
||||
for name in required:
|
||||
fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=stage_fd)
|
||||
try:
|
||||
write_all(fd, payloads[name])
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.fsync(stage_fd)
|
||||
os.close(stage_fd)
|
||||
stage_fd = None
|
||||
require_base_identity()
|
||||
try:
|
||||
os.stat(output_name, dir_fd=base_fd, follow_symlinks=False)
|
||||
fail("unsafe output path")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.rename(extract_stage, output_name, src_dir_fd=base_fd, dst_dir_fd=base_fd)
|
||||
extract_stage = None
|
||||
published = True
|
||||
os.fsync(base_fd)
|
||||
require_base_identity()
|
||||
except Exception as error:
|
||||
if isinstance(error, RuntimeError):
|
||||
print(str(error), file=sys.stderr)
|
||||
else:
|
||||
print("extraction operational failure (details redacted)", file=sys.stderr)
|
||||
sys.exit_code = 1
|
||||
finally:
|
||||
if stage_fd is not None:
|
||||
os.close(stage_fd)
|
||||
if extract_stage is not None and base_fd is not None:
|
||||
try:
|
||||
remove_anchored_directory(extract_stage)
|
||||
except Exception:
|
||||
sys.exit_code = 1
|
||||
if published and getattr(sys, "exit_code", 0) and base_fd is not None:
|
||||
try:
|
||||
remove_anchored_directory(output_name)
|
||||
except Exception:
|
||||
pass
|
||||
if archive_fd is not None:
|
||||
os.close(archive_fd)
|
||||
if archive_stage is not None and base_fd is not None:
|
||||
try:
|
||||
os.unlink(archive_stage, dir_fd=base_fd)
|
||||
os.fsync(base_fd)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if base_fd is not None:
|
||||
os.close(base_fd)
|
||||
if getattr(sys, "exit_code", 0):
|
||||
raise SystemExit(sys.exit_code)
|
||||
PY
|
||||
`;}
|
||||
async function writeCommands(repo,root){const commands=join(root,"commands");for(const [name,body]of [...httpCommands(root),["render-1.sh",renderCommand(repo,root,1)],["render-2.sh",renderCommand(repo,root,2)],["diff-rendered.sh",`#!/bin/sh\nset -eu\ndiff -u ${quote(join(root,"rendered/runtime-1.yaml"))} ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["config-check.sh",`#!/bin/sh\nset -eu\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-1.yaml"))}\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["git-inspect.sh",`#!/bin/sh\nset -eu\ncommit=\${1:?published commit required}\ncase "$commit" in *[!0-9a-f]*|'') exit 2;; esac\n[ \${#commit} -eq 40 ] || exit 2\ngit -C ${quote(join(root,"installation/registry/repo"))} log --oneline --decorate -10 "$commit"\ngit -C ${quote(join(root,"installation/registry/repo"))} ls-tree -r "$commit" -- workspaces workspace-content\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspaces/p1-filesystem.yaml"\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspace-content/p1-filesystem/evidence/guide.md"\n`],["extract-export.sh",extractCommand(repo,root)],["secret-scan.sh",`#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root=${quote(root)}
|
||||
node --input-type=module - "$root" <<'NODE'
|
||||
import { spawnSync } from "node:child_process";import { constants } from "node:fs";import { lstat, open, readdir } from "node:fs/promises";import { join, relative } from "node:path";
|
||||
const root=process.argv[2],randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");let found=false,filesystemCount=0,filesystemTotal=0,nameTotal=0;const gitDirs=new Set([join(root,"remote.git")]);const containsCanary=value=>randomized.test(value)||value.includes(fixed);const finding=kind=>{console.error("secret canary found in "+kind+" (path and value redacted)");found=true;};
|
||||
async function maybeGitDir(path){try{const head=await lstat(join(path,"HEAD")),objects=await lstat(join(path,"objects"));if(head.isFile()&&objects.isDirectory()&&!head.isSymbolicLink()&&!objects.isSymbolicLink())gitDirs.add(path);}catch{}}
|
||||
async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){if(++filesystemCount>200000)throw Error("bound");const child=join(path,entry.name),rel=relative(root,child),nameBytes=Buffer.from(entry.name),pathBytes=Buffer.from(rel);if(nameBytes.length>255||pathBytes.length>4096||(nameTotal+=nameBytes.length+pathBytes.length)>67108864)throw Error("bound");if(containsCanary(nameBytes.toString("latin1"))||containsCanary(pathBytes.toString("latin1")))finding("filesystem name bytes");if(entry.isSymbolicLink()){console.error("unsafe symlink during secret scan (path redacted)");found=true;continue;}if(entry.isDirectory()){if(entry.name===".git")await maybeGitDir(child);if(rel==="fixture-secrets")continue;await walk(child);continue;}if(!entry.isFile())throw Error("unsupported");let handle;try{handle=await open(child,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat();if(!before.isFile()||before.size>33554432)throw Error("bound");filesystemTotal+=before.size;if(filesystemTotal>1073741824)throw Error("bound");const bytes=await handle.readFile(),after=await handle.stat();if(bytes.length!==before.size||after.dev!==before.dev||after.ino!==before.ino||after.size!==before.size)throw Error("changed");const value=bytes.toString("latin1"),allowedRequest=rel==="requests/invalid-credential.json"&&value.includes(fixed)&&!randomized.test(value);if(containsCanary(value)&&!allowedRequest)finding("filesystem bytes");}finally{if(handle)await handle.close();}}}
|
||||
function gitRun(args,options={}){const result=spawnSync("git",args,{...options,stdio:[options.input===undefined?"ignore":"pipe","pipe","pipe"]});if(result.error||result.status!==0)throw Error("git");return result.stdout;}
|
||||
function scanGit(gitDir){const listing=gitRun(["--git-dir",gitDir,"cat-file","--batch-all-objects","--unordered","--batch-check=%(objectname) %(objecttype) %(objectsize)"],{encoding:"utf8",maxBuffer:16*1024*1024}).trim(),objects=listing?listing.split("\\n"):[];if(objects.length>100000)throw Error("bound");let total=0;for(const line of objects){const match=line.match(/^([0-9a-f]{40,64}) (blob|commit|tree|tag) (\\d+)$/);if(!match)throw Error("git");const[,oid,type,sizeText]=match,size=Number(sizeText);total+=size;if(!Number.isSafeInteger(size)||size>33554432||total>536870912)throw Error("bound");const raw=gitRun(["--git-dir",gitDir,"cat-file",type,oid],{maxBuffer:Math.max(1024,size+1)});if(raw.length!==size)throw Error("changed");if(containsCanary(raw.toString("latin1")))finding(type==="blob"?"Git blob":"Git object");}}
|
||||
try{await walk(root);for(const gitDir of gitDirs)scanGit(gitDir);if(found)process.exitCode=1;else console.log("no fixture secret canary outside fixture-secrets or in any bounded Git object");}catch{console.error("secret scan operational failure (details redacted)");process.exitCode=2;}
|
||||
NODE
|
||||
`],["absence-check.sh",`#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root=${quote(root)}
|
||||
node --input-type=module - "$root" <<'NODE'
|
||||
import { readdir } from "node:fs/promises";import { join,relative } from "node:path";
|
||||
const root=process.argv[2];let count=0,rejected=false;const artifactName=name=>name.toUpperCase()==="ACTIVE"||/(?:materiali[sz](?:e|ed|ation)|preprocess|embedding|qdrant|retention)/i.test(name);
|
||||
async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){if(++count>200000)throw Error("bound");const child=join(path,entry.name),parts=relative(root,child).split("/");if(parts.some((part,index)=>part==="artifacts"&&parts[index+1]==="evidence")||artifactName(entry.name))rejected=true;if(entry.isSymbolicLink())continue;if(entry.isDirectory()&&entry.name!==".git")await walk(child);}}
|
||||
try{await walk(root);if(rejected){console.error("unexpected out-of-scope P2+ artifact (path redacted)");process.exitCode=1;}else console.log("no out-of-scope runtime artifact found");}catch{console.error("out-of-scope artifact check failed safely (details redacted)");process.exitCode=2;}
|
||||
NODE
|
||||
`]]){await atomicWrite(join(commands,name),body,0o700);await chmod(join(commands,name),0o700);}}
|
||||
export async function prepareManual(options={}){
|
||||
const unknown=Object.keys(options).filter(key=>!["repositoryRoot","skipBuild"].includes(key));if(unknown.length)throw new Error(`unknown or automated-run prepare input: ${unknown.join(", ")}`);
|
||||
const{repositoryRoot=defaultRepositoryRoot,skipBuild=false}=options,repo=realpathSync(repositoryRoot),root=fixedManualRoot(repo),lifecycle=await acquireLifecycle(repo,"prepare");let entryBinding,ownershipCreated=false;
|
||||
try{
|
||||
await requireLifecycleContext(lifecycle);await checkPrerequisites(repo);if(!skipBuild)await run("npm",["--prefix",join(repo,"backend"),"run","build"]);await requireLifecycleContext(lifecycle);
|
||||
entryBinding=await readBoundEntrypoint(repo);const entrypoint=entryBinding.identity;await entryBinding.handle.close();entryBinding=undefined;
|
||||
noSymlinkExisting(repo,root);try{await mkdir(root,{recursive:false,mode:0o700});}catch(error){if(error.code==="EEXIST")throw new Error("manual acceptance root already exists; stop/cleanup it explicitly");throw error;}await bindLifecycleRoot(lifecycle,root);
|
||||
for(const path of ["installation/registry","installation/data","installation/runtime","fixture-secrets","fixtures/descriptors","requests","responses","exports/raw","exports/extracted","rendered","logs","commands"]){await mkdir(join(root,path),{recursive:true,mode:path==="fixture-secrets"?0o700:0o755});await requireLifecycleContext(lifecycle,{root:true});}
|
||||
const distManifestValue=await buildDistManifest(repo),distManifestRecord=await exclusiveRecord(join(root,"installation/runtime/backend-dist.manifest.json"),distManifestValue,"production distribution manifest"),distManifest={path:distManifestRecord.path,dev:distManifestRecord.dev,ino:distManifestRecord.ino,size:distManifestRecord.bytes.length,sha256:createHash("sha256").update(distManifestRecord.bytes).digest("hex")};await requireLifecycleContext(lifecycle,{root:true});
|
||||
const nonce=randomBytes(32).toString("hex"),createdAt=new Date().toISOString();await exclusiveRecord(join(root,"ownership.json"),ownedValue(repo,root,nonce,{entrypoint,distManifest,createdAt}),"manual ownership");ownershipCreated=true;await requireLifecycleContext(lifecycle,{root:true});
|
||||
const backendLogPath=join(root,"logs/backend.log"),backendLogHandle=await open(backendLogPath,"wx",0o600);let backendLogEntry;try{await backendLogHandle.chmod(0o600);await backendLogHandle.sync();backendLogEntry=await backendLogHandle.stat();}finally{await backendLogHandle.close();}directorySync(dirname(backendLogPath));const backendLog={path:backendLogPath,dev:backendLogEntry.dev,ino:backendLogEntry.ino};
|
||||
await requireLifecycleContext(lifecycle,{root:true});
|
||||
try{await initializeGit(root);}catch(error){try{await requireLifecycleContext(lifecycle,{root:true});}catch{await cleanupFailedPrepare(repo,lifecycle);throw new Error("manual acceptance parent or root identity changed during prepare");}throw error;}await requireLifecycleContext(lifecycle,{root:true});
|
||||
const items=descriptors();for(const workspace of items)await atomicWrite(join(root,"fixtures/descriptors",`${workspace.workspace.id}.json`),`${JSON.stringify(workspace,null,2)}\n`);
|
||||
const secrets={"dwh-password":`DWH-${randomBytes(16).toString("hex")}`,"evidence-signed-urls.json":JSON.stringify([`https://evidence.example.test/guide.md?token=SIGNED-${randomBytes(16).toString("hex")}`]),"evidence-access":`ACCESS-${randomBytes(16).toString("hex")}`,"evidence-secret":`SECRET-${randomBytes(16).toString("hex")}`,"evidence-session":`SESSION-${randomBytes(16).toString("hex")}`};for(const[name,value]of Object.entries(secrets))await atomicWrite(join(root,"fixture-secrets",name),value,0o600);
|
||||
const env={};for(const workspace of items){const ns=workspace.workspace.id.toUpperCase().replaceAll("-","_"),prefix=`THT_WS_${ns}`;Object.assign(env,{[`${prefix}_DWH_TRANSPORT`]:"postgres_direct",[`${prefix}_DWH_HOST`]:"dwh.invalid",[`${prefix}_DWH_PORT`]:"5432",[`${prefix}_DWH_USER`]:"reader",[`${prefix}_DWH_PASSWORD_FILE`]:join(root,"fixture-secrets/dwh-password")});}Object.assign(env,{THT_WORKSPACE_SECRET_ROOTS:join(root,"fixture-secrets"),THT_WS_P1_HTTP_EVIDENCE_SIGNED_URLS_FILE:join(root,"fixture-secrets/evidence-signed-urls.json"),THT_WS_P1_S3_EVIDENCE_ACCESS_KEY_FILE:join(root,"fixture-secrets/evidence-access"),THT_WS_P1_S3_EVIDENCE_SECRET_KEY_FILE:join(root,"fixture-secrets/evidence-secret"),THT_WS_P1_S3_EVIDENCE_SESSION_TOKEN_FILE:join(root,"fixture-secrets/evidence-session")});
|
||||
await atomicWrite(join(root,"installation/bindings.env"),Object.entries(env).map(([k,v])=>`${k}=${quote(v)}`).join("\n")+"\n");await atomicWrite(join(root,"installation/base.yaml"),"{}\n");for(const[name,value]of Object.entries(requestFixtures(items)))await atomicWrite(join(root,"requests",name),`${JSON.stringify(value,null,2)}\n`);await writeCommands(repo,root);await atomicWrite(join(root,"GUIDE.md"),guide(repo,root),0o600);await requireLifecycleContext(lifecycle,{root:true});await atomicWrite(join(root,"ownership.json"),`${JSON.stringify(ownedValue(repo,root,nonce,{backendLog,entrypoint,distManifest,stage:"READY",createdAt}),null,2)}\n`);await requireLifecycleContext(lifecycle,{root:true});return{repositoryRoot:repo,root,nonce};
|
||||
}catch(error){
|
||||
if(entryBinding)await entryBinding.handle.close().catch(()=>{});
|
||||
if(ownershipCreated){try{await requireLifecycleContext(lifecycle,{root:true});}catch{await cleanupFailedPrepare(repo,lifecycle).catch(()=>{});throw new Error("manual acceptance parent or root identity changed during prepare");}}
|
||||
throw error;
|
||||
}finally{await removeExactRecord(lifecycle);}
|
||||
}
|
||||
function portAvailable(port,label=`${HOST}:${port}`){return new Promise((resolvePromise,reject)=>{const server=net.createServer();server.once("error",error=>error.code==="EADDRINUSE"?reject(new Error(`${label} is occupied`)):reject(error));server.listen({host:HOST,port,exclusive:true},()=>server.close(()=>resolvePromise()));});}
|
||||
async function requireCanonicalDirectory(path,label){const entry=await lstat(path);if(!entry.isDirectory()||entry.isSymbolicLink()||await realpath(path)!==path)throw new Error(`${label} directory identity is unsafe`);return entry;}
|
||||
async function requireAbsent(path,label){try{await lstat(path);throw new Error(`${label} is legacy or unsafe`);}catch(error){if(error.code!=="ENOENT")throw error;}}
|
||||
async function validateServeFilesystem(repo,root,owned){
|
||||
if(root!==fixedManualRoot(repo))throw new Error("owned root identity is unsafe");
|
||||
for(const [path,label] of [
|
||||
[repo,"repository root"],[join(repo,".artifacts"),"artifact root"],[join(repo,".artifacts/manual-acceptance"),"manual root ancestor"],[root,"owned root"],
|
||||
[join(root,"installation"),"owned installation"],[join(root,"installation/runtime"),"owned runtime"],[join(root,"installation/data"),"owned data"],
|
||||
[join(root,"installation/registry"),"owned registry"],[join(root,"fixture-secrets"),"owned secrets"],[join(root,"logs"),"owned logs"],
|
||||
[join(repo,"backend"),"backend root"],[join(repo,"backend/dist"),"backend distribution"],
|
||||
])await requireCanonicalDirectory(path,label);
|
||||
await requireAbsent(legacySupervisorPath(root),"legacy supervisor");
|
||||
if(owned.stage!=="READY")throw new Error("manual acceptance preparation is incomplete");
|
||||
const script=join(repo,"backend/dist/server.js");await requireEntrypointPathIdentity(owned.entrypoint);const manifestRecord=await readBoundDistManifest(repo,owned);try{await validateDistFiles(repo,manifestRecord.files);}finally{await manifestRecord.handle.close();}
|
||||
const logPath=join(root,"logs/backend.log");
|
||||
if(owned.backendLog?.path!==logPath)throw new Error("backend log ownership identity is unsafe");
|
||||
return{script,logPath};
|
||||
}
|
||||
function openOwnedBackendLog(owned,logPath){
|
||||
if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("backend log no-follow protection is unavailable");
|
||||
let fd;
|
||||
try{
|
||||
fd=openSync(logPath,constants.O_WRONLY|constants.O_APPEND|constants.O_NOFOLLOW);
|
||||
const entry=fstatSync(fd),pathEntry=lstatSync(logPath);
|
||||
if(!entry.isFile()||(entry.mode&0o777)!==0o600||entry.nlink!==1||entry.dev!==owned.backendLog.dev||entry.ino!==owned.backendLog.ino||pathEntry.isSymbolicLink()||!pathEntry.isFile()||pathEntry.dev!==entry.dev||pathEntry.ino!==entry.ino)throw new Error("backend log identity is unsafe");
|
||||
return fd;
|
||||
}catch(error){if(fd!==undefined)closeSync(fd);throw error;}
|
||||
}
|
||||
async function ensureRuntimeDirectory(path){try{await mkdir(path,{mode:0o700});}catch(error){if(error.code!=="EEXIST")throw error;}const entry=await requireCanonicalDirectory(path,"owned runtime child");if((entry.mode&0o077)!==0)throw new Error("owned runtime child mode is unsafe");}
|
||||
async function processStart(pid){return (await run("ps",["-p",String(pid),"-o","lstart="])).stdout.trim();}
|
||||
async function processArgs(pid){return (await run("ps",["-ww","-p",String(pid),"-o","command="])).stdout.trim();}
|
||||
async function processCwd(pid){try{return await realpath(`/proc/${pid}/cwd`);}catch{try{const out=(await run("lsof",["-a","-p",String(pid),"-d","cwd","-Fn"])).stdout.split("\n").find(x=>x.startsWith("n"));return out?await realpath(out.slice(1)):"";}catch{return"";}}}
|
||||
async function processExecutable(pid){try{return await realpath(`/proc/${pid}/exe`);}catch{try{const paths=(await run("lsof",["-a","-p",String(pid),"-d","txt","-Fn"])).stdout.split("\n").filter(x=>x.startsWith("n")).map(x=>x.slice(1));for(const path of paths){try{const canonical=await realpath(path);if(canonical===realpathSync(process.execPath))return canonical;}catch{}}return"";}catch{return"";}}}
|
||||
function alive(pid){try{process.kill(pid,0);return true;}catch{return false;}}
|
||||
async function readPid(root){const path=join(root,"backend.pid"),entry=await lstat(path);if(!entry.isFile()||entry.isSymbolicLink()||(entry.mode&0o777)!==0o600)throw new Error("backend PID record is unsafe");const bytes=await readFile(path,"utf8");let value;try{value=JSON.parse(bytes);}catch{throw new Error("backend PID record is malformed");}return{path,bytes,value,dev:entry.dev,ino:entry.ino};}
|
||||
async function validateProcess(repo,root,owned,pidRecord){
|
||||
const script=join(repo,"backend/dist/server.js"),entrypoint=owned.entrypoint;
|
||||
if(pidRecord.schemaVersion!==1||pidRecord.kind!=="p1-manual-backend"||pidRecord.status!=="RUNNING"||!Number.isSafeInteger(pidRecord.pid)||pidRecord.pid<2||!HEX64.test(pidRecord.reservationNonce??"")||pidRecord.nonce!==owned.nonce||pidRecord.root!==root||pidRecord.repositoryRoot!==repo||pidRecord.executable!==process.execPath||pidRecord.preload!==PRELOAD||pidRecord.script!==script||JSON.stringify(pidRecord.entrypoint)!==JSON.stringify(entrypoint)||!pidRecord.startIdentity||pidRecord.control?.host!==HOST||pidRecord.control?.port!==CONTROL_PORT)throw new Error("backend process identity mismatch; refusing cooperative control");
|
||||
await requireEntrypointPathIdentity(entrypoint);const manifestRecord=await readBoundDistManifest(repo,owned);try{await validateDistFiles(repo,manifestRecord.files);}finally{await manifestRecord.handle.close();}if(!alive(pidRecord.pid))throw new Error("backend PID is stale; operator inspection required");
|
||||
const[start,args,cwd,executable]=await Promise.all([processStart(pidRecord.pid),processArgs(pidRecord.pid),processCwd(pidRecord.pid),processExecutable(pidRecord.pid)]);
|
||||
const expectedArgs=[pidRecord.executable,"--import",pidRecord.preload,pidRecord.script,`--p1-manual-nonce=${owned.nonce}`,`--p1-root=${root}`,`--p1-control-nonce=${pidRecord.reservationNonce}`,`--p1-entry-sha256=${entrypoint.sha256}`,`--p1-entry-dev=${entrypoint.dev}`,`--p1-entry-ino=${entrypoint.ino}`].join(" ");
|
||||
if(start!==pidRecord.startIdentity||cwd!==repo||executable!==realpathSync(pidRecord.executable)||args!==expectedArgs)throw new Error("backend process identity mismatch; refusing cooperative control");return true;
|
||||
}
|
||||
async function waitForChildExit(child,milliseconds){if(!child||child.exitCode!==null||child.signalCode!==null)return true;return await Promise.race([new Promise(resolvePromise=>child.once("exit",()=>resolvePromise(true))),new Promise(resolvePromise=>setTimeout(()=>resolvePromise(child.exitCode!==null||child.signalCode!==null),milliseconds))]);}
|
||||
async function healthStatus(){return await new Promise((resolvePromise,reject)=>{const request=http.get({host:HOST,port:PORT,path:"/health",timeout:500},response=>{const status=response.statusCode;response.resume();response.once("end",()=>resolvePromise(status));});request.once("timeout",()=>request.destroy(new Error("backend health readiness timeout")));request.once("error",reject);});}
|
||||
function exactControlIdentity(answer,child,owned,root,reservationNonce){return answer?.pid===child.pid&&answer?.nonce===owned.nonce&&answer?.controlNonce===reservationNonce&&answer?.root===root&&answer?.control?.host===HOST&&answer?.control?.port===CONTROL_PORT;}
|
||||
function exactOwnedListener(answer,generation){return answer?.listener?.listening===true&&answer.listener.host===HOST&&answer.listener.port===PORT&&Number.isSafeInteger(answer.listener.generation)&&answer.listener.generation>0&&(generation===undefined||answer.listener.generation===generation);}
|
||||
export async function serveManual({repositoryRoot=defaultRepositoryRoot,beforeSpawn}={}){
|
||||
const repo=realpathSync(repositoryRoot),lifecycle=await acquireLifecycle(repo,"serve");let pidRecord,child,controlObserved=false,logFd,entryBinding,manifestBinding;
|
||||
try{
|
||||
const owned=await readManualOwnership({repositoryRoot:repo}),root=owned.root;await bindLifecycleRoot(lifecycle,root);
|
||||
if(owned.listener.host!==HOST||owned.listener.port!==PORT)throw new Error("non-loopback or unexpected bind refused");
|
||||
const{script,logPath}=await validateServeFilesystem(repo,root,owned);await requireLifecycleContext(lifecycle,{root:true});
|
||||
logFd=openOwnedBackendLog(owned,logPath);entryBinding=await readBoundEntrypoint(repo);if(JSON.stringify(entryBinding.identity)!==JSON.stringify(owned.entrypoint))throw new Error("production entrypoint identity changed");manifestBinding=await readBoundDistManifest(repo,owned);
|
||||
const reservationNonce=randomBytes(32).toString("hex");pidRecord=await exclusiveRecord(join(root,"backend.pid"),{schemaVersion:1,kind:"p1-manual-backend-reservation",status:"RESERVED",reservationNonce,nonce:owned.nonce,root,repositoryRoot:repo},"backend PID record");
|
||||
await Promise.all([portAvailable(PORT),portAvailable(CONTROL_PORT,`${HOST}:${CONTROL_PORT} control port`)]);await requireLifecycleContext(lifecycle,{root:true});
|
||||
await ensureRuntimeDirectory(join(root,"installation/runtime/home"));await ensureRuntimeDirectory(join(root,"installation/runtime/tmp"));await ensureRuntimeDirectory(join(root,"installation/runtime/tht-home"));await requireLifecycleContext(lifecycle,{root:true});
|
||||
const inherited={};for(const key of ["PATH","LANG","LC_ALL","TZ"])if(process.env[key]!==undefined)inherited[key]=process.env[key];
|
||||
const env={...inherited,HOME:join(root,"installation/runtime/home"),TMPDIR:join(root,"installation/runtime/tmp"),HOST,PORT:String(PORT),AUTH_MODE:"none",THT_BIN:join(repo,"harness/.venv/bin/tht"),THT_HARNESS_DIR:join(repo,"harness"),THT_DATA_ROOT:join(root,"installation/data"),SETTINGS_FILE:join(root,"installation/data/settings.json"),MAINTENANCE_STATE_FILE:join(root,"installation/data/maintenance.json"),THT_WORKSPACE_REGISTRY_ROOT:join(root,"installation/registry"),THT_WORKSPACE_GIT_REMOTE:join(root,"remote.git"),THT_WORKSPACE_GIT_BRANCH:"main",THT_WORKSPACE_GIT_AUTHOR_NAME:"P1 Manual API Publisher",THT_WORKSPACE_GIT_AUTHOR_EMAIL:"p1-manual-api@example.invalid",THT_WORKSPACE_INSTALLATION_ID:"p1-manual-acceptance",THT_WORKSPACE_SECRET_ROOTS:join(root,"fixture-secrets"),THT_HOME:join(root,"installation/runtime/tht-home")};
|
||||
if(beforeSpawn)await beforeSpawn({script,entrypoint:{...owned.entrypoint}});await requireLifecycleContext(lifecycle,{root:true});
|
||||
const entryArgs=[`--p1-entry-sha256=${owned.entrypoint.sha256}`,`--p1-entry-dev=${owned.entrypoint.dev}`,`--p1-entry-ino=${owned.entrypoint.ino}`];
|
||||
child=spawn(process.execPath,["--import",PRELOAD,script,`--p1-manual-nonce=${owned.nonce}`,`--p1-root=${root}`,`--p1-control-nonce=${reservationNonce}`,...entryArgs],{cwd:repo,env,detached:true,stdio:["ignore",logFd,logFd,entryBinding.handle.fd,manifestBinding.handle.fd]});
|
||||
await entryBinding.handle.close();entryBinding=undefined;await manifestBinding.handle.close();manifestBinding=undefined;closeSync(logFd);logFd=undefined;
|
||||
let start="";for(let n=0;n<80;n++){if(child.exitCode!==null)break;try{start=await processStart(child.pid);if(start)break;}catch{}await new Promise(r=>setTimeout(r,25));}
|
||||
if(!start)throw new Error("backend failed before process identity could be recorded");await requireLifecycleContext(lifecycle,{root:true});
|
||||
pidRecord=await replaceExactRecord(pidRecord,{schemaVersion:1,kind:"p1-manual-backend-reservation",status:"STARTING",pid:child.pid,reservationNonce,nonce:owned.nonce,root,repositoryRoot:repo,executable:process.execPath,preload:PRELOAD,script,entrypoint:owned.entrypoint,startIdentity:start,control:{host:HOST,port:CONTROL_PORT}});
|
||||
const deadline=Date.now()+7000;let readyAnswer,listenerGeneration;
|
||||
while(Date.now()<deadline&&child.exitCode===null){
|
||||
let status;try{status=await controlRequest({host:HOST,port:CONTROL_PORT},{action:"status",nonce:reservationNonce});}catch{await new Promise(r=>setTimeout(r,50));continue;}
|
||||
if(!exactControlIdentity(status,child,owned,root,reservationNonce)||status.status!=="STARTING")throw new Error("backend control status identity mismatch");controlObserved=true;
|
||||
if(!exactOwnedListener(status)){await new Promise(r=>setTimeout(r,50));continue;}listenerGeneration=status.listener.generation;
|
||||
await requireLifecycleContext(lifecycle,{root:true});await requireEntrypointPathIdentity(owned.entrypoint);
|
||||
let httpCode;try{httpCode=await healthStatus();}catch{await new Promise(r=>setTimeout(r,50));continue;}if(!Number.isSafeInteger(httpCode)||httpCode<200||httpCode>=300)throw new Error(`backend health readiness returned HTTP ${httpCode}`);
|
||||
readyAnswer=await controlRequest({host:HOST,port:CONTROL_PORT},{action:"ready",nonce:reservationNonce});if(!exactControlIdentity(readyAnswer,child,owned,root,reservationNonce)||readyAnswer.status!=="READY"||!exactOwnedListener(readyAnswer,listenerGeneration))throw new Error("backend READY listener acknowledgement identity mismatch");
|
||||
const finalHealth=await healthStatus();if(!Number.isSafeInteger(finalHealth)||finalHealth<200||finalHealth>=300)throw new Error("backend final health readiness failed");
|
||||
const finalStatus=await controlRequest({host:HOST,port:CONTROL_PORT},{action:"status",nonce:reservationNonce});if(!exactControlIdentity(finalStatus,child,owned,root,reservationNonce)||finalStatus.status!=="READY"||!exactOwnedListener(finalStatus,listenerGeneration))throw new Error("backend final listener identity mismatch");readyAnswer=finalStatus;break;
|
||||
}
|
||||
if(!readyAnswer)throw new Error("backend readiness failed; inspect owned backend log and starting PID record");
|
||||
const runningValue={schemaVersion:1,kind:"p1-manual-backend",status:"RUNNING",pid:child.pid,reservationNonce,nonce:owned.nonce,root,repositoryRoot:repo,executable:process.execPath,preload:PRELOAD,script,entrypoint:owned.entrypoint,startIdentity:start,control:{host:HOST,port:CONTROL_PORT},listener:{host:HOST,port:PORT,generation:listenerGeneration}};
|
||||
await requireLifecycleContext(lifecycle,{root:true});await validateProcess(repo,root,owned,runningValue);if(child.exitCode!==null||!alive(child.pid))throw new Error("backend exited before RUNNING publication");pidRecord=await replaceExactRecord(pidRecord,runningValue);await requireLifecycleContext(lifecycle,{root:true});
|
||||
const publishedStatus=await controlRequest(runningValue.control,{action:"status",nonce:reservationNonce});if(!exactControlIdentity(publishedStatus,child,owned,root,reservationNonce)||publishedStatus.status!=="READY"||!exactOwnedListener(publishedStatus,listenerGeneration)){await removeExactRecord(pidRecord);throw new Error("backend listener changed during RUNNING publication");}
|
||||
child.unref();return child.pid;
|
||||
}catch(error){
|
||||
if(entryBinding)await entryBinding.handle.close().catch(()=>{});if(manifestBinding)await manifestBinding.handle.close().catch(()=>{});if(logFd!==undefined){closeSync(logFd);logFd=undefined;}
|
||||
if(child&&controlObserved){try{const value=pidRecord?JSON.parse(pidRecord.bytes):undefined;await controlRequest({host:HOST,port:CONTROL_PORT},{action:"stop",nonce:value?.reservationNonce});}catch{}await waitForChildExit(child,3000);}else if(child)await waitForChildExit(child,8500);
|
||||
if(pidRecord&&(!child||child.exitCode!==null||child.signalCode!==null||!alive(child.pid)))await removeExactRecord(pidRecord).catch(()=>{});throw error;
|
||||
}finally{if(entryBinding)await entryBinding.handle.close().catch(()=>{});if(manifestBinding)await manifestBinding.handle.close().catch(()=>{});if(logFd!==undefined)closeSync(logFd);await removeExactRecord(lifecycle);}
|
||||
}
|
||||
export async function stopManual({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),lifecycle=await acquireLifecycle(repo,"stop");try{const owned=await readManualOwnership({repositoryRoot:repo}),root=owned.root;if(owned.stage!=="READY")throw new Error("owned backend was never prepared");await bindLifecycleRoot(lifecycle,root);let record;try{record=await readPid(root);}catch(error){if(error.code==="ENOENT")throw new Error("owned backend is not served");throw error;}await requireLifecycleContext(lifecycle,{root:true});await validateProcess(repo,root,owned,record.value);const answer=await controlRequest(record.value.control,{action:"stop",nonce:record.value.reservationNonce});if(answer.status!=="STOPPING"||answer.pid!==record.value.pid||answer.nonce!==owned.nonce||answer.controlNonce!==record.value.reservationNonce||answer.root!==root||answer.control?.host!==HOST||answer.control?.port!==CONTROL_PORT)throw new Error("backend cooperative stop acknowledgement mismatch; PID record retained");for(let n=0;n<100;n++){if(!alive(record.value.pid)){await requireLifecycleContext(lifecycle,{root:true});await removeExactRecord(record);await requireLifecycleContext(lifecycle,{root:true});return;}await new Promise(r=>setTimeout(r,100));}throw new Error("owned backend did not stop cooperatively; operator must intervene; PID record retained");}finally{await removeExactRecord(lifecycle);}}
|
||||
const ANCHORED_REMOVE_SOURCE=String.raw`import os,stat,sys
|
||||
parent,parent_dev,parent_ino,root_dev,root_ino,tomb=sys.argv[1:]
|
||||
pfd=rfd=None
|
||||
def die(): raise RuntimeError("anchored cleanup refused")
|
||||
def clear(fd):
|
||||
names=os.listdir(fd)
|
||||
if len(names)>200000: die()
|
||||
for name in names:
|
||||
if name in (".",".."): die()
|
||||
item=os.stat(name,dir_fd=fd,follow_symlinks=False)
|
||||
if stat.S_ISDIR(item.st_mode):
|
||||
child=os.open(name,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd)
|
||||
try: clear(child)
|
||||
finally: os.close(child)
|
||||
os.rmdir(name,dir_fd=fd)
|
||||
elif stat.S_ISREG(item.st_mode) or stat.S_ISLNK(item.st_mode): os.unlink(name,dir_fd=fd)
|
||||
else: die()
|
||||
try:
|
||||
pfd=os.open(parent,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
||||
ps=os.fstat(pfd)
|
||||
if (ps.st_dev,ps.st_ino)!=(int(parent_dev),int(parent_ino)): die()
|
||||
rfd=os.open("p1",os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=pfd)
|
||||
rs=os.fstat(rfd)
|
||||
if (rs.st_dev,rs.st_ino)!=(int(root_dev),int(root_ino)): die()
|
||||
try: os.stat(tomb,dir_fd=pfd,follow_symlinks=False); die()
|
||||
except FileNotFoundError: pass
|
||||
os.rename("p1",tomb,src_dir_fd=pfd,dst_dir_fd=pfd);os.fsync(pfd)
|
||||
clear(rfd);os.close(rfd);rfd=None;os.rmdir(tomb,dir_fd=pfd);os.fsync(pfd)
|
||||
except Exception:
|
||||
print("anchored cleanup refused (details redacted)",file=sys.stderr);raise SystemExit(1)
|
||||
finally:
|
||||
if rfd is not None: os.close(rfd)
|
||||
if pfd is not None: os.close(pfd)
|
||||
`;
|
||||
async function anchoredRemoveOwnedRoot(lifecycle,owned){const tomb=`.deleting-p1-${owned.nonce.slice(0,16)}`;try{await run("python3",["-c",ANCHORED_REMOVE_SOURCE,lifecycle.parentPath,String(lifecycle.parentEntry.dev),String(lifecycle.parentEntry.ino),String(lifecycle.rootEntry.dev),String(lifecycle.rootEntry.ino),tomb]);}catch{throw new Error("anchored cleanup refused; owned identities changed");}}
|
||||
export async function cleanupManual({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),lifecycle=await acquireLifecycle(repo,"cleanup");try{const owned=await readManualOwnership({repositoryRoot:repo}),root=owned.root;await bindLifecycleRoot(lifecycle,root);try{const record=await readPid(root);if(record.value.status==="RUNNING"&&alive(record.value.pid)){await validateProcess(repo,root,owned,record.value);throw new Error("owned backend is live; run stop first");}throw new Error("stale or starting backend PID record requires operator inspection and stop validation");}catch(error){if(error.code!=="ENOENT")throw error;}if(root!==fixedManualRoot(repo)||!below(join(repo,".artifacts"),root))throw new Error("cleanup root identity mismatch");await requireLifecycleContext(lifecycle,{root:true});await anchoredRemoveOwnedRoot(lifecycle,owned);await requireLifecycleContext(lifecycle);}finally{await removeExactRecord(lifecycle);}}
|
||||
async function main(){const[action,...rest]=process.argv.slice(2);if(rest.length||!["prepare","serve","stop","cleanup"].includes(action??""))throw new Error("usage: p1-manual-acceptance.mjs prepare|serve|stop|cleanup");if(action==="prepare")await prepareManual();if(action==="serve")await serveManual();if(action==="stop")await stopManual();if(action==="cleanup")await cleanupManual();console.log(`P1 manual acceptance ${action}: ${action==="prepare"?"PENDING":"complete"}`);}
|
||||
if(process.argv[1]&&realpathSync(process.argv[1])===modulePath)main().catch(error=>{console.error(`p1 manual acceptance refused: ${error.message}`);process.exitCode=1;});
|
||||
@@ -1,787 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile, spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, lstat, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import net from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
import {
|
||||
cleanupManual, fixedManualRoot, prepareManual, readManualOwnership, serveManual, stopManual,
|
||||
} from "./p1-manual-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepo() {
|
||||
const root = await realpath(await mkdtemp(join(tmpdir(), "p1-manual-repo-")));
|
||||
roots.push(root);
|
||||
for (const path of ["scripts/p1-acceptance.sh", "scripts/test-p1-acceptance.sh", "backend/scripts/p1-acceptance.mjs", "backend/dist/server.js"]) {
|
||||
await mkdir(dirname(join(root, path)), { recursive: true });
|
||||
await writeFile(join(root, path), path.endsWith(".sh") ? "#!/bin/sh\n" : "export {};\n", { mode: 0o700 });
|
||||
}
|
||||
await symlink(new URL("../node_modules", import.meta.url).pathname, join(root, "backend", "node_modules"), "dir");
|
||||
await mkdir(join(root, "harness", ".venv", "bin"), { recursive: true });
|
||||
await writeFile(join(root, "harness", ".venv", "bin", "tht"), "#!/bin/sh\n", { mode: 0o700 });
|
||||
await chmod(join(root, "harness", ".venv", "bin", "tht"), 0o700);
|
||||
await mkdir(join(root, "harness", "workspaces"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
test.afterEach(async () => Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))));
|
||||
|
||||
test("prepare refuses a pre-existing or symlink fixed root", async () => {
|
||||
const repo = await fakeRepo(); const root = fixedManualRoot(repo);
|
||||
await mkdir(root, { recursive: true });
|
||||
await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /already exists/);
|
||||
await rm(root, { recursive: true });
|
||||
const target = `${root}-target`; await mkdir(target, { recursive: true }); await symlink(target, root);
|
||||
await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /already exists|symlink/);
|
||||
});
|
||||
|
||||
test("prepare requires Task 8 and prerequisites before creating state", async () => {
|
||||
const repo = await fakeRepo(); await rm(join(repo, "scripts", "p1-acceptance.sh"));
|
||||
await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /Task 8/);
|
||||
await assert.rejects(lstat(fixedManualRoot(repo)));
|
||||
});
|
||||
|
||||
test("public wrapper exposes only four actions and rejects automated-run prepare input", async () => {
|
||||
const wrapper=new URL("../../scripts/p1-manual-acceptance.sh",import.meta.url),source=await readFile(wrapper,"utf8");
|
||||
assert.match(source,/prepare\|serve\|stop\|cleanup/); assert.doesNotMatch(source,/integration\|automated|prepare\|serve\|stop\|cleanup\|/);
|
||||
await assert.rejects(execFileAsync("bash",[wrapper.pathname,"prepare",".artifacts/p1-integration/run"]),error=>error.code===2&&/usage:/.test(error.stderr));
|
||||
});
|
||||
|
||||
test("prepare rejects unknown automated-run input before creating its root", async () => {
|
||||
const repo = await fakeRepo();
|
||||
await assert.rejects(
|
||||
prepareManual({ repositoryRoot: repo, skipBuild: true, automatedRun: join(repo, ".artifacts", "p1-integration") }),
|
||||
/unknown|automated/i,
|
||||
);
|
||||
await assert.rejects(lstat(fixedManualRoot(repo)));
|
||||
});
|
||||
|
||||
test("prepare requires the non-Task-8 tht prerequisite before creating state", async () => {
|
||||
const repo = await fakeRepo(); await rm(join(repo, "harness", ".venv", "bin", "tht"));
|
||||
await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /missing prerequisite.*tht/);
|
||||
await assert.rejects(lstat(fixedManualRoot(repo)));
|
||||
});
|
||||
|
||||
test("prepare and permanent docs declare the python3 extractor prerequisite", async () => {
|
||||
const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8"),docs=await readFile(new URL("../../docs/testing/p1-manual-acceptance.md",import.meta.url),"utf8");
|
||||
assert.match(source,/for\(const command of \[.*["']python3["']/s); assert.match(docs,/python3/);
|
||||
});
|
||||
|
||||
test("prepare creates independent pending topology, fixtures, commands and guide without verdict", async () => {
|
||||
const repo = await fakeRepo(); const run = await prepareManual({ repositoryRoot: repo, skipBuild: true });
|
||||
assert.equal(run.root, fixedManualRoot(repo));
|
||||
const owned = await readManualOwnership({ repositoryRoot: repo });
|
||||
assert.equal(owned.status, "PENDING"); assert.equal(owned.listener.host, "127.0.0.1"); assert.equal(owned.listener.port, 8791);
|
||||
for (const path of ["remote.git/HEAD", "author/.git", "installation/registry", "fixture-secrets/dwh-password", "fixtures/descriptors/p1-filesystem.json", "requests/status.json", "responses", "exports", "rendered", "logs", "commands/render-1.sh", "commands/render-2.sh", "logs/backend.log", "GUIDE.md"]) await lstat(join(run.root, path));
|
||||
await assert.rejects(lstat(join(run.root, "VERDICT.md")));
|
||||
const guide = await readFile(join(run.root, "GUIDE.md"), "utf8");
|
||||
let previous = -1; for (let n = 1; n <= 14; n++) { const at = guide.indexOf(`${n}. `); assert.ok(at > previous, `step ${n} ordered`); previous = at; }
|
||||
assert.doesNotMatch(guide, /cat .*fixture-secrets|show.*secret contents/i); for(const id of ["p1-filesystem","p1-http","p1-s3"])assert.match(guide,new RegExp(`extract-export\\.sh[^\\n]+${id}`));
|
||||
for(const contract of [/stable repository-root/,/ownership-first recovery/,/one production Node PID owns both/,/opened no-follow descriptor/,/arbitrary `.git` repository/,/name\/path bytes/,/artifacts\/evidence/,/opened no-follow `rendered` directory/])assert.match(guide,contract);
|
||||
const traversal=JSON.parse(await readFile(join(run.root,"requests","invalid-traversal.json"),"utf8")); assert.match(traversal.workspace.evidence.source.uri,/\.\./);
|
||||
const bindings=await readFile(join(run.root,"installation","bindings.env"),"utf8"); assert.match(bindings,new RegExp(`^THT_WORKSPACE_SECRET_ROOTS=.*fixture-secrets`,"m")); const scan=await readFile(join(run.root,"commands","secret-scan.sh"),"utf8"),extract=await readFile(join(run.root,"commands","extract-export.sh"),"utf8"); assert.match(scan,/batch-all-objects/); assert.match(scan,/cat-file/); assert.match(scan,/maybeGitDir/); assert.match(extract,/ZIP contains a symlink or nonregular entry/);
|
||||
const pubFs=await readFile(join(run.root,"commands","http-05-publish-p1-filesystem.sh"),"utf8"),pubHttp=await readFile(join(run.root,"commands","http-06-publish-p1-http.sh"),"utf8"),pubS3=await readFile(join(run.root,"commands","http-07-publish-p1-s3.sh"),"utf8"); assert.match(pubFs,/responses\/status\.json/); assert.match(pubHttp,/responses\/publish-p1-filesystem\.json/); assert.match(pubS3,/responses\/publish-p1-http\.json/); assert.doesNotMatch(pubFs,/REPLACE_WITH/);
|
||||
const render = await readFile(join(run.root, "commands", "render-1.sh"), "utf8");
|
||||
for(const name of await readdir(join(run.root,"commands")))if(name.endsWith(".sh"))await execFileAsync("bash",["-n",join(run.root,"commands",name)]);
|
||||
assert.match(render, /read-p1-filesystem\.json/); assert.match(render, /responses\/pull\.json/); assert.doesNotMatch(render, /responses\/publish-p1-filesystem\.json/); assert.match(render, /snapshotPath/); assert.match(render, /p1-render-snapshot\.mjs/);
|
||||
});
|
||||
|
||||
test("cleanup rejects unowned, live, mismatched and symlink state and preserves siblings", async () => {
|
||||
const repo = await fakeRepo(); const integration = join(repo, ".artifacts", "p1-integration"); const sibling = join(repo, ".artifacts", "manual-acceptance", "foreign");
|
||||
await mkdir(integration, { recursive: true }); await writeFile(join(integration, "sentinel"), "keep");
|
||||
await mkdir(sibling, { recursive: true }); await writeFile(join(sibling, "sentinel"), "keep");
|
||||
await assert.rejects(cleanupManual({ repositoryRoot: repo }), /ownership|root/);
|
||||
const run = await prepareManual({ repositoryRoot: repo, skipBuild: true });
|
||||
const ownershipPath = join(run.root, "ownership.json"); const owned = JSON.parse(await readFile(ownershipPath)); owned.root += "-wrong"; await writeFile(ownershipPath, JSON.stringify(owned));
|
||||
await assert.rejects(cleanupManual({ repositoryRoot: repo }), /identity/); assert.equal((await lstat(run.root)).isDirectory(), true);
|
||||
assert.equal(await readFile(join(integration, "sentinel"), "utf8"), "keep"); assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "keep");
|
||||
});
|
||||
|
||||
test("cleanup removes only the exact stopped owned root and never creates verdict", async () => {
|
||||
const repo = await fakeRepo(); const run = await prepareManual({ repositoryRoot: repo, skipBuild: true });
|
||||
await cleanupManual({ repositoryRoot: repo }); await assert.rejects(lstat(run.root));
|
||||
});
|
||||
|
||||
|
||||
async function installFakeServer(repo, { startupDelay = 0, healthStatus = 200, marker } = {}) {
|
||||
await writeFile(join(repo, "backend", "dist", "server.js"), `import http from "node:http";
|
||||
${marker ? `import { writeFileSync } from "node:fs"; writeFileSync(${JSON.stringify(marker)}, "executed");` : ""}
|
||||
const server=http.createServer((req,res)=>{res.statusCode=req.url==="/health"?${healthStatus}:200;res.setHeader("content-type","application/json");res.end(JSON.stringify({status:"ok",ambient:process.env.THT_DWH_API_KEY,maintenance:process.env.MAINTENANCE_STATE_FILE,wrongMaintenance:process.env.THT_MAINTENANCE_STATE_FILE}));});
|
||||
setTimeout(()=>server.listen(Number(process.env.PORT),process.env.HOST),${startupDelay});
|
||||
`);
|
||||
}
|
||||
|
||||
|
||||
async function matchingManualServerPids(root, nonce) {
|
||||
const { stdout } = await execFileAsync("ps", ["ax", "-o", "pid=,command="]);
|
||||
const nonceArg = `--p1-manual-nonce=${nonce}`, rootArg = `--p1-root=${root}`;
|
||||
return stdout.split("\n").filter(line => line.includes(nonceArg) && line.includes(rootArg))
|
||||
.map(line => Number(line.trim().match(/^(\d+)/)?.[1])).filter(Number.isSafeInteger);
|
||||
}
|
||||
async function listenerPids() {
|
||||
try {
|
||||
const { stdout } = await execFileAsync("lsof", ["-nP", "-t", "-iTCP:8791", "-sTCP:LISTEN"]);
|
||||
return [...new Set(stdout.trim().split("\n").filter(Boolean).map(Number))];
|
||||
} catch (error) {
|
||||
if (error.code === 1) return [];
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
test("prepare and cleanup share one external lifecycle lock for the whole transaction", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim();
|
||||
const bin=join(repo,"blocking-bin"),entered=join(repo,"prepare-entered"),release=join(repo,"prepare-release");
|
||||
await mkdir(bin); await writeFile(join(bin,"git"),`#!/bin/sh
|
||||
if [ "$1" = init ] && [ ! -e ${JSON.stringify(entered)} ]; then
|
||||
: > ${JSON.stringify(entered)}
|
||||
n=0
|
||||
while [ ! -e ${JSON.stringify(release)} ] && [ "$n" -lt 250 ]; do sleep 0.02; n=$((n + 1)); done
|
||||
[ -e ${JSON.stringify(release)} ] || exit 99
|
||||
fi
|
||||
exec ${JSON.stringify(realGit)} "$@"
|
||||
`,{mode:0o700});
|
||||
const priorPath=process.env.PATH; process.env.PATH=`${bin}:${priorPath}`;
|
||||
try {
|
||||
const preparing=prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
for(let n=0;n<200;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,10));}
|
||||
await lstat(entered);
|
||||
const lock=join(repo,".p1-manual-acceptance.lifecycle.lock");
|
||||
const lockEntry=await lstat(lock); assert.equal(lockEntry.isFile(),true); assert.equal(lockEntry.mode&0o777,0o600);
|
||||
const lockBytes=await readFile(lock,"utf8"),lockValue=JSON.parse(lockBytes);
|
||||
assert.deepEqual(Object.keys(lockValue).sort(),["kind","lifecycleNonce","operation","repositoryRoot","root","schemaVersion"].sort());
|
||||
assert.equal(lockValue.kind,"p1-manual-lifecycle"); assert.equal(lockValue.operation,"prepare");
|
||||
assert.match(lockValue.lifecycleNonce,/^[0-9a-f]{64}$/); assert.equal(lockValue.repositoryRoot,repo); assert.equal(lockValue.root,fixedManualRoot(repo));
|
||||
assert.equal(lockBytes,`${JSON.stringify(lockValue,null,2)}\n`);
|
||||
await assert.rejects(cleanupManual({repositoryRoot:repo}),/lifecycle lock.*exists|operator inspection/i);
|
||||
await writeFile(release,"go"); const run=await preparing;
|
||||
await readManualOwnership({repositoryRoot:repo}); await assert.rejects(lstat(lock));
|
||||
await cleanupManual({repositoryRoot:repo}); await assert.rejects(lstat(run.root));
|
||||
} finally { process.env.PATH=priorPath; await writeFile(release,"go").catch(()=>{}); }
|
||||
});
|
||||
|
||||
test("the external lifecycle lock prevents old-root/new-root ABA and ownership is read only under lock", async () => {
|
||||
const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8");
|
||||
assert.match(source,/\.p1-manual-acceptance\.lifecycle\.lock/);
|
||||
const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const lockPath=join(repo,".p1-manual-acceptance.lifecycle.lock"),nonce="f".repeat(64),bytes=`${nonce}\n`;
|
||||
const handle=await open(lockPath,"wx",0o600); await handle.writeFile(bytes); await handle.sync();
|
||||
try {
|
||||
const old=JSON.parse(await readFile(join(run.root,"ownership.json"),"utf8"));
|
||||
await rm(run.root,{recursive:true}); await mkdir(run.root,{recursive:true});
|
||||
await writeFile(join(run.root,"ownership.json"),JSON.stringify({...old,nonce:"e".repeat(64)}),{mode:0o600});
|
||||
for(const operation of [serveManual,stopManual,cleanupManual]){
|
||||
await assert.rejects(operation({repositoryRoot:repo}),/lifecycle lock.*exists|operator inspection/i);
|
||||
assert.equal((await lstat(run.root)).isDirectory(),true);
|
||||
}
|
||||
} finally { await handle.close(); await rm(lockPath,{force:true}); }
|
||||
});
|
||||
|
||||
test("external lifecycle lock release preserves an exact-byte inode replacement", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim();
|
||||
const bin=join(repo,"replacement-bin"),entered=join(repo,"replacement-entered"),release=join(repo,"replacement-release");
|
||||
await mkdir(bin); await writeFile(join(bin,"git"),`#!/bin/sh
|
||||
if [ "$1" = init ] && [ ! -e ${JSON.stringify(entered)} ]; then
|
||||
: > ${JSON.stringify(entered)}
|
||||
n=0
|
||||
while [ ! -e ${JSON.stringify(release)} ] && [ "$n" -lt 250 ]; do sleep 0.02; n=$((n + 1)); done
|
||||
[ -e ${JSON.stringify(release)} ] || exit 99
|
||||
fi
|
||||
exec ${JSON.stringify(realGit)} "$@"
|
||||
`,{mode:0o700});
|
||||
const priorPath=process.env.PATH; process.env.PATH=`${bin}:${priorPath}`; let preparing;
|
||||
try {
|
||||
preparing=prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
for(let n=0;n<200;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,10));}
|
||||
await lstat(entered);
|
||||
const lock=join(repo,".p1-manual-acceptance.lifecycle.lock"),bytes=await readFile(lock);
|
||||
const original=await lstat(lock),replacement=join(dirname(lock),".replacement-lifecycle-lock");
|
||||
await writeFile(replacement,bytes,{mode:0o600}); const replacementEntry=await lstat(replacement);
|
||||
assert.notEqual(replacementEntry.ino,original.ino); await rename(replacement,lock); await writeFile(release,"go");
|
||||
await assert.rejects(preparing,/lifecycle record.*unsafe|lifecycle record.*changed|operator inspection/i); preparing=undefined;
|
||||
const retained=await lstat(lock); assert.equal(retained.dev,replacementEntry.dev); assert.equal(retained.ino,replacementEntry.ino);
|
||||
assert.deepEqual(await readFile(lock),bytes);
|
||||
} finally {
|
||||
process.env.PATH=priorPath; await writeFile(release,"go").catch(()=>{}); if(preparing)await preparing.catch(()=>{});
|
||||
}
|
||||
});
|
||||
|
||||
test("prepare records one regular 0600 backend log and no generated supervisor", async () => {
|
||||
const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const owned=await readManualOwnership({repositoryRoot:repo}),entry=await lstat(join(run.root,"logs/backend.log"));
|
||||
assert.equal(entry.isFile(),true); assert.equal(entry.isSymbolicLink(),false); assert.equal(entry.mode&0o777,0o600);
|
||||
assert.deepEqual(owned.backendLog,{path:join(run.root,"logs/backend.log"),dev:entry.dev,ino:entry.ino});
|
||||
await assert.rejects(lstat(join(run.root,"installation/runtime/p1-backend-supervisor.mjs")));
|
||||
});
|
||||
|
||||
// A delayed real listener leaves the pre-fix port-check/spawn window open long enough for every
|
||||
// overlapping call. The backend.pid reservation, rather than scheduler timing, must pick one owner.
|
||||
test("concurrent serves reserve one exact process and leave no orphan after stop", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo,{startupDelay:400});
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const owned=await readManualOwnership({repositoryRoot:repo}); let winner;
|
||||
try {
|
||||
const results=await Promise.allSettled(Array.from({length:12},()=>serveManual({repositoryRoot:repo})));
|
||||
const fulfilled=results.filter(result=>result.status==="fulfilled");
|
||||
assert.equal(fulfilled.length,1,`one serve fulfills: ${results.map(result=>result.status).join(",")}`);
|
||||
assert.equal(results.filter(result=>result.status==="rejected").length,11);
|
||||
winner=fulfilled[0].value;
|
||||
const pidPath=join(run.root,"backend.pid"),record=JSON.parse(await readFile(pidPath,"utf8")),entry=await lstat(pidPath);
|
||||
assert.equal(entry.mode&0o777,0o600); assert.equal(record.status,"RUNNING");
|
||||
assert.match(record.reservationNonce,/^[0-9a-f]{64}$/); assert.equal(record.pid,winner);
|
||||
assert.equal(record.nonce,owned.nonce); assert.equal(record.root,run.root); assert.equal(record.repositoryRoot,repo);
|
||||
assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[winner]);
|
||||
assert.deepEqual(await listenerPids(),[winner]); assert.doesNotThrow(()=>process.kill(winner,0));
|
||||
await stopManual({repositoryRoot:repo});
|
||||
await assert.rejects(lstat(pidPath)); assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]);
|
||||
assert.deepEqual(await listenerPids(),[]); assert.throws(()=>process.kill(winner,0));
|
||||
await cleanupManual({repositoryRoot:repo}); await assert.rejects(lstat(run.root));
|
||||
} finally {
|
||||
for(const pid of await matchingManualServerPids(run.root,owned.nonce))try{process.kill(pid,"SIGTERM");}catch{}
|
||||
await new Promise(resolvePromise=>setTimeout(resolvePromise,50));
|
||||
for(const pid of await matchingManualServerPids(run.root,owned.nonce))try{process.kill(pid,"SIGKILL");}catch{}
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("cooperative stop is serialized and production never sends a numeric terminating signal", { concurrency: false }, async () => {
|
||||
const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8");
|
||||
assert.doesNotMatch(source,/process\.kill\([^,]+,\s*["']SIG(?:TERM|KILL|INT)/);
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const owned=await readManualOwnership({repositoryRoot:repo}); const pid=await serveManual({repositoryRoot:repo});
|
||||
const record=JSON.parse(await readFile(join(run.root,"backend.pid"),"utf8")); assert.equal(record.control.host,"127.0.0.1");
|
||||
const unauthorized=await new Promise((resolvePromise,reject)=>{const socket=net.createConnection(record.control),timer=setTimeout(()=>socket.destroy(new Error("control timeout")),1000);let bytes="";socket.setEncoding("utf8");socket.on("connect",()=>socket.end(JSON.stringify({action:"stop",nonce:"0".repeat(64)})));socket.on("data",chunk=>bytes+=chunk);socket.on("error",reject);socket.on("close",()=>{clearTimeout(timer);resolvePromise(bytes);});});
|
||||
assert.equal(unauthorized,""); assert.doesNotThrow(()=>process.kill(pid,0));
|
||||
const stopped=await Promise.allSettled([stopManual({repositoryRoot:repo}),stopManual({repositoryRoot:repo})]);
|
||||
assert.equal(stopped.filter(result=>result.status==="fulfilled").length,1);
|
||||
assert.equal(stopped.filter(result=>result.status==="rejected").length,1);
|
||||
await assert.rejects(lstat(join(run.root,"backend.pid"))); assert.deepEqual(await listenerPids(),[]);
|
||||
assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); assert.throws(()=>process.kill(pid,0));
|
||||
await cleanupManual({repositoryRoot:repo});
|
||||
});
|
||||
|
||||
test("serve binds the one fixed loopback address, refuses a second PID, and guarded stop removes identity", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const priorAmbient=process.env.THT_DWH_API_KEY; process.env.THT_DWH_API_KEY="AMBIENT-MUST-NOT-PASS"; const pid=await serveManual({repositoryRoot:repo}); assert.equal(Number.isSafeInteger(pid),true);
|
||||
const health=await (await fetch("http://127.0.0.1:8791/health")).json(); assert.equal(health.status,"ok"); assert.equal(health.ambient,undefined); assert.equal(health.wrongMaintenance,undefined); assert.equal(health.maintenance,join(run.root,"installation/data/maintenance.json")); if(priorAmbient===undefined)delete process.env.THT_DWH_API_KEY;else process.env.THT_DWH_API_KEY=priorAmbient;
|
||||
await assert.rejects(serveManual({repositoryRoot:repo}),/PID record/);
|
||||
await stopManual({repositoryRoot:repo}); await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
await assert.rejects(fetch("http://127.0.0.1:8791/health",{signal:AbortSignal.timeout(200)}));
|
||||
await cleanupManual({repositoryRoot:repo});
|
||||
});
|
||||
|
||||
test("serve requires a 2xx HTTP health check and leaves no orphan on 503", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo,{healthStatus:503}); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const owned=await readManualOwnership({repositoryRoot:repo});
|
||||
await assert.rejects(serveManual({repositoryRoot:repo}),/health|readiness/i);
|
||||
await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
assert.deepEqual(await listenerPids(),[]); assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]);
|
||||
});
|
||||
|
||||
test("serve launches exact server.js with immutable preload and fixed owned control port", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const pid=await serveManual({repositoryRoot:repo}),record=JSON.parse(await readFile(join(run.root,"backend.pid"),"utf8"));
|
||||
assert.equal(record.pid,pid); assert.equal(record.script,join(repo,"backend/dist/server.js"));
|
||||
assert.equal(record.control.host,"127.0.0.1"); assert.equal(record.control.port,8792);
|
||||
assert.match(record.preload,/^data:text\/javascript;base64,/);
|
||||
const args=(await execFileAsync("ps",["-ww","-p",String(pid),"-o","command="])).stdout.trim();
|
||||
assert.equal(args,[process.execPath,"--import",record.preload,record.script,`--p1-manual-nonce=${record.nonce}`,`--p1-root=${run.root}`,`--p1-control-nonce=${record.reservationNonce}`,`--p1-entry-sha256=${record.entrypoint.sha256}`,`--p1-entry-dev=${record.entrypoint.dev}`,`--p1-entry-ino=${record.entrypoint.ino}`].join(" "));
|
||||
await stopManual({repositoryRoot:repo});
|
||||
});
|
||||
|
||||
test("serve refuses legacy supervisor, runtime, server and log substitutions before code or outside writes", { concurrency: false }, async () => {
|
||||
for(const kind of ["legacy-supervisor","runtime-symlink","server-symlink","log-symlink","log-replaced"]){
|
||||
const repo=await fakeRepo(),marker=join(repo,`outside-${kind}.marker`); await installFakeServer(repo,{marker});
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),outside=join(repo,`outside-${kind}`); await mkdir(outside);
|
||||
if(kind==="legacy-supervisor")await symlink(join(outside,"outside.mjs"),join(run.root,"installation/runtime/p1-backend-supervisor.mjs"));
|
||||
if(kind==="runtime-symlink"){await rm(join(run.root,"installation/runtime"),{recursive:true});await symlink(outside,join(run.root,"installation/runtime"));}
|
||||
if(kind==="server-symlink"){
|
||||
const external=join(outside,"server.js"); await writeFile(external,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(marker)},"bad");`);
|
||||
await rm(join(repo,"backend/dist/server.js")); await symlink(external,join(repo,"backend/dist/server.js"));
|
||||
}
|
||||
if(kind==="log-symlink"){await rm(join(run.root,"logs/backend.log"));await symlink(join(outside,"captured.log"),join(run.root,"logs/backend.log"));}
|
||||
if(kind==="log-replaced"){await rm(join(run.root,"logs/backend.log"));await writeFile(join(run.root,"logs/backend.log"),"",{mode:0o600});}
|
||||
await assert.rejects(serveManual({repositoryRoot:repo}),/unsafe|identity|symlink|legacy|realpath|log/i,kind);
|
||||
await assert.rejects(lstat(marker),undefined,`${kind} must refuse before server execution`);
|
||||
assert.deepEqual(await readdir(outside),kind==="server-symlink"?["server.js"]:[]);
|
||||
await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("serve refuses an occupied fixed control port before spawning", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),marker=join(repo,"server-executed"); await installFakeServer(repo,{marker}); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const blocker=net.createServer(); await new Promise((resolvePromise,reject)=>blocker.once("error",reject).listen(8792,"127.0.0.1",resolvePromise));
|
||||
try { await assert.rejects(serveManual({repositoryRoot:repo}),/8792.*occupied|control.*occupied/); } finally { await new Promise(resolvePromise=>blocker.close(resolvePromise)); }
|
||||
await assert.rejects(lstat(marker)); await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
});
|
||||
|
||||
test("serve refuses an occupied fixed port and never creates a PID or verdict", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const blocker=net.createServer(); await new Promise((resolvePromise,reject)=>blocker.once("error",reject).listen(8791,"127.0.0.1",resolvePromise));
|
||||
try { await assert.rejects(serveManual({repositoryRoot:repo}),/occupied/); } finally { await new Promise(resolvePromise=>blocker.close(resolvePromise)); }
|
||||
await assert.rejects(lstat(join(run.root,"backend.pid"))); await assert.rejects(lstat(join(run.root,"VERDICT.md")));
|
||||
});
|
||||
|
||||
test("serve and cleanup refuse stale or mismatched PID records without signaling", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
await writeFile(join(run.root,"backend.pid"),JSON.stringify({pid:999999,nonce:"wrong"}),{mode:0o600});
|
||||
await assert.rejects(serveManual({repositoryRoot:repo}),/PID record/);
|
||||
await assert.rejects(stopManual({repositoryRoot:repo}),/identity mismatch/);
|
||||
await assert.rejects(cleanupManual({repositoryRoot:repo}),/identity|stale/);
|
||||
assert.equal((await lstat(run.root)).isDirectory(),true);
|
||||
});
|
||||
|
||||
test("serve refuses non-loopback ownership without creating process state", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const ownershipPath=join(run.root,"ownership.json"),owned=JSON.parse(await readFile(ownershipPath,"utf8"));
|
||||
owned.listener.host="0.0.0.0"; await writeFile(ownershipPath,JSON.stringify(owned));
|
||||
await assert.rejects(serveManual({repositoryRoot:repo}),/identity|loopback|bind/);
|
||||
await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
});
|
||||
|
||||
test("cleanup refuses a correctly owned live server until guarded stop", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const pid=await serveManual({repositoryRoot:repo});
|
||||
try {
|
||||
await assert.rejects(cleanupManual({repositoryRoot:repo}),/owned backend is live|stop first/);
|
||||
assert.doesNotThrow(()=>process.kill(pid,0));
|
||||
} finally {
|
||||
try { await stopManual({repositoryRoot:repo}); } catch { try { process.kill(pid,"SIGTERM"); } catch {} }
|
||||
}
|
||||
await cleanupManual({repositoryRoot:repo}); await assert.rejects(lstat(run.root));
|
||||
});
|
||||
|
||||
async function processStartIdentity(pid) {
|
||||
return (await execFileAsync("ps",["-p",String(pid),"-o","lstart="])).stdout.trim();
|
||||
}
|
||||
async function stopTestProcess(child) {
|
||||
if (child.exitCode === null) child.kill("SIGTERM");
|
||||
if (child.exitCode === null) await new Promise(resolvePromise=>child.once("exit",resolvePromise));
|
||||
}
|
||||
|
||||
test("live foreign executable, cwd, start and args mismatches are never signaled", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const owned=JSON.parse(await readFile(join(run.root,"ownership.json"),"utf8"));
|
||||
const script=join(run.root,"installation/runtime/p1-backend-supervisor.mjs"),nonceArg=`--p1-manual-nonce=${owned.nonce}`,rootArg=`--p1-root=${run.root}`,reservationNonce="a".repeat(64),controlArg=`--p1-control-nonce=${reservationNonce}`;
|
||||
await writeFile(script,"setInterval(()=>{},1000);\n",{mode:0o600});
|
||||
const cases=[
|
||||
["executable",()=>spawn("bash",["-c","while :; do sleep 1; done",script,nonceArg,rootArg,controlArg],{cwd:repo,stdio:"ignore"}),{}],
|
||||
["cwd",()=>spawn(process.execPath,[script,nonceArg,rootArg,controlArg],{cwd:tmpdir(),stdio:"ignore"}),{}],
|
||||
["start",()=>spawn(process.execPath,[script,nonceArg,rootArg,controlArg],{cwd:repo,stdio:"ignore"}),{startIdentity:"foreign-start"}],
|
||||
["args",()=>spawn(process.execPath,[script],{cwd:repo,stdio:"ignore"}),{}],
|
||||
];
|
||||
for(const [name,start,override] of cases){
|
||||
const child=start();
|
||||
try {
|
||||
let actualStart=""; for(let n=0;n<50&&!actualStart;n++){try{actualStart=await processStartIdentity(child.pid);}catch{} if(!actualStart)await new Promise(r=>setTimeout(r,20));}
|
||||
assert.ok(actualStart,`live ${name} process started`);
|
||||
const record={schemaVersion:1,kind:"p1-manual-backend",status:"RUNNING",pid:child.pid,reservationNonce,nonce:owned.nonce,root:run.root,repositoryRoot:repo,executable:process.execPath,script,startIdentity:actualStart,control:{host:"127.0.0.1",port:1},...override};
|
||||
await writeFile(join(run.root,"backend.pid"),JSON.stringify(record),{mode:0o600});
|
||||
await assert.rejects(stopManual({repositoryRoot:repo}),/process identity mismatch|refusing cooperative control/);
|
||||
assert.doesNotThrow(()=>process.kill(child.pid,0));
|
||||
await rm(join(run.root,"backend.pid"));
|
||||
} finally { await stopTestProcess(child); await rm(join(run.root,"backend.pid"),{force:true}); }
|
||||
}
|
||||
});
|
||||
|
||||
test("generated render command validates saved responses and owned snapshot before renderer", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const script=join(run.root,"commands/render-1.sh"), output=join(run.root,"rendered/runtime-1.yaml");
|
||||
const invoke=()=>execFileAsync("bash",[script],{cwd:repo});
|
||||
await assert.rejects(invoke(),/saved response is missing/);
|
||||
await writeFile(join(run.root,"responses/read-p1-filesystem.json"),"{"); await writeFile(join(run.root,"responses/pull.json"),"{}");
|
||||
await assert.rejects(invoke(),/malformed JSON/);
|
||||
const a="a".repeat(40),b="b".repeat(40),outside=join(repo,"outside.yaml"); await writeFile(outside,"x");
|
||||
await writeFile(join(run.root,"responses/read-p1-filesystem.json"),JSON.stringify({revision:{commit:a,snapshotPath:outside}})); await writeFile(join(run.root,"responses/pull.json"),JSON.stringify({head:b}));
|
||||
await assert.rejects(invoke(),/revisions differ/);
|
||||
await writeFile(join(run.root,"responses/pull.json"),JSON.stringify({head:a})); await assert.rejects(invoke(),/snapshot escapes/);
|
||||
await assert.rejects(lstat(output));
|
||||
});
|
||||
|
||||
const renderSnapshotYaml=`workspace:
|
||||
schema_version: 3
|
||||
id: p1-filesystem
|
||||
name: P1 filesystem
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: public
|
||||
supported_transports: [postgres_direct]
|
||||
semantic_index:
|
||||
vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
|
||||
embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
|
||||
llm_policy:
|
||||
allowed: [zai/glm-5.2]
|
||||
evidence:
|
||||
source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
|
||||
policy: {max_chunk_chars: 4000, retain_published_generations: 3}
|
||||
`;
|
||||
|
||||
test("generated render command binds snapshot bytes to the commit manifest and Git blob end to end", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const author=join(run.root,"author"); await mkdir(join(author,"workspaces"),{recursive:true});
|
||||
await writeFile(join(author,"workspaces","p1-filesystem.yaml"),renderSnapshotYaml);
|
||||
await execFileAsync("git",["add","workspaces"],{cwd:author}); await execFileAsync("git",["commit","-m","publish p1"],{cwd:author}); await execFileAsync("git",["push","origin","main"],{cwd:author});
|
||||
const commit=(await execFileAsync("git",["rev-parse","HEAD"],{cwd:author})).stdout.trim();
|
||||
const blob=(await execFileAsync("git",["rev-parse","HEAD:workspaces/p1-filesystem.yaml"],{cwd:author})).stdout.trim();
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]);
|
||||
const commitDir=join(run.root,"installation/registry/snapshots",commit); await mkdir(commitDir,{recursive:true});
|
||||
const snapshot=join(commitDir,"p1-filesystem.yaml"),snapshotPath=snapshot,snapshotSha=sha256(renderSnapshotYaml);
|
||||
await writeFile(snapshot,renderSnapshotYaml);
|
||||
const readPath=join(run.root,"responses/read-p1-filesystem.json"),pullPath=join(run.root,"responses/pull.json"),script=join(run.root,"commands/render-1.sh"),script2=join(run.root,"commands/render-2.sh"),output=join(run.root,"rendered/runtime-1.yaml"),output2=join(run.root,"rendered/runtime-2.yaml");
|
||||
const rendererStub=join(repo,"backend/scripts/p1-render-snapshot.mjs"),stubArgs=join(run.root,"rendered/stub-args.json");
|
||||
await writeFile(rendererStub,`import { writeFileSync } from "node:fs";\nwriteFileSync(${JSON.stringify(stubArgs)}, JSON.stringify(process.argv.slice(2)));\n`);
|
||||
const revision={id:"p1-filesystem",commit,blob,snapshotPath};
|
||||
const manifest=(entry=revision)=>({head:commit,revisions:[entry],files:{"p1-filesystem.yaml":snapshotSha}});
|
||||
await writeFile(readPath,JSON.stringify({revision})); await writeFile(pullPath,JSON.stringify({head:commit}));
|
||||
await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest.*(missing|unbounded)/i);
|
||||
await assert.rejects(lstat(output));
|
||||
const legacyRevision={...revision}; legacyRevision[["st","ate"].join("")]=["oper","ational"].join("");
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest(legacyRevision)));
|
||||
await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest revision is invalid/);
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest({...revision,unexpected:"field"})));
|
||||
await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest revision is invalid/);
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest()));
|
||||
await execFileAsync("bash",[script],{cwd:repo});
|
||||
assert.deepEqual(JSON.parse(await readFile(stubArgs,"utf8")),["--ownership",join(run.root,"ownership.json"),"--snapshot",snapshot,"--output",output,"--snapshot-sha256",snapshotSha]);
|
||||
await writeFile(snapshot,renderSnapshotYaml.replace("max_chunk_chars: 4000","max_chunk_chars: 3999"));
|
||||
await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/snapshot bytes differ from manifest digest/);
|
||||
await assert.rejects(lstat(output2));
|
||||
await writeFile(snapshot,renderSnapshotYaml);
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify({...manifest(),head:"c".repeat(40)}));
|
||||
await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/snapshot manifest identity is invalid/);
|
||||
await assert.rejects(lstat(output2));
|
||||
for(const malformed of [
|
||||
{id:"p1-filesystem",commit,blob},
|
||||
{...revision,id:"p1-http"},
|
||||
{...revision,commit:"c".repeat(40)},
|
||||
{...revision,blob:"f".repeat(39)},
|
||||
{...revision,blob:[blob]},
|
||||
{...revision,snapshotPath:join(commitDir,"wrong.yaml")},
|
||||
]){
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest(malformed)));
|
||||
await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/snapshot manifest revision is invalid/);
|
||||
}
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest({...revision,blob:"f".repeat(40)})));
|
||||
await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/saved revision blob differs from snapshot manifest/);
|
||||
await assert.rejects(lstat(output2));
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest()));
|
||||
await writeFile(readPath,JSON.stringify({revision:{...revision,blob:"f".repeat(40)}}));
|
||||
await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/saved revision blob differs/);
|
||||
await assert.rejects(lstat(output2));
|
||||
});
|
||||
|
||||
|
||||
const sha256=bytes=>createHash("sha256").update(bytes).digest("hex");
|
||||
async function makeExportZip(directory,name,{payloads={},manifest,workspaceId="p1-filesystem",extra=false,symlinkReadme=false}={}) {
|
||||
const source=join(directory,`${name}-source`),zip=join(directory,`${name}.zip`); await mkdir(source,{recursive:true});
|
||||
const files={"workspace.yaml":`workspace:\n id: ${workspaceId}\n`,"contract.env.example":"SAFE=path\n","README.md":"# Safe\n",...payloads};
|
||||
const value=manifest??{schema_version:1,workspace_id:workspaceId,files:Object.fromEntries(Object.entries(files).map(([n,b])=>[n,sha256(b)]))};
|
||||
await writeFile(join(source,"manifest.json"),JSON.stringify(value));
|
||||
for(const [file,bytes] of Object.entries(files))if(!(symlinkReadme&&file==="README.md"))await writeFile(join(source,file),bytes);
|
||||
if(symlinkReadme)await symlink("workspace.yaml",join(source,"README.md"));
|
||||
if(extra)await writeFile(join(source,"extra.txt"),"extra");
|
||||
const names=["manifest.json","workspace.yaml","contract.env.example","README.md",...(extra?["extra.txt"]:[])];
|
||||
await execFileAsync("zip",["-q",...(symlinkReadme?["-y"]:[]),zip,...names],{cwd:source}); return zip;
|
||||
}
|
||||
|
||||
test("generated ZIP verifier enforces exact manifest mapping, hashes, entries and regular files", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh");
|
||||
const invoke=async(name,options={})=>execFileAsync("bash",[extract,await makeExportZip(run.root,name,options),join(run.root,"exports/extracted",name),"p1-filesystem"],{cwd:repo});
|
||||
await invoke("valid");
|
||||
const safe={"workspace.yaml":"workspace:\n id: p1-filesystem\n","contract.env.example":"SAFE=path\n","README.md":"# Safe\n"};
|
||||
const hashes=Object.fromEntries(Object.entries(safe).map(([n,b])=>[n,sha256(b)]));
|
||||
await assert.rejects(invoke("missing-map",{manifest:{schema_version:1,workspace_id:"p1-filesystem",files:{"workspace.yaml":hashes["workspace.yaml"],"contract.env.example":hashes["contract.env.example"]}}}),/manifest/i);
|
||||
await assert.rejects(invoke("short-hash",{manifest:{schema_version:1,workspace_id:"p1-filesystem",files:{...hashes,"README.md":"abc"}}}),/manifest/i);
|
||||
await assert.rejects(invoke("extra-entry",{extra:true}),/unsafe-zip/);
|
||||
await assert.rejects(invoke("nonregular",{symlinkReadme:true}),/symlink|nonregular/);
|
||||
});
|
||||
|
||||
test("generated ZIP verifier binds identity, stages source once, and rejects symlink output ancestry", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh");
|
||||
for(const id of ["p1-filesystem","p1-http","p1-s3"]){
|
||||
const zip=await makeExportZip(run.root,`valid-${id}`,{workspaceId:id});
|
||||
await execFileAsync("bash",[extract,zip,join(run.root,"exports/extracted",id),id],{cwd:repo});
|
||||
}
|
||||
const wrong=await makeExportZip(run.root,"wrong-valid-id",{workspaceId:"p1-http"});
|
||||
await assert.rejects(execFileAsync("bash",[extract,wrong,join(run.root,"exports/extracted/wrong-id"),"p1-s3"],{cwd:repo}),/workspace.*identity|workspace_id/i);
|
||||
const descriptorMismatch=await makeExportZip(run.root,"descriptor-mismatch",{workspaceId:"p1-http",payloads:{"workspace.yaml":"workspace:\n id: p1-s3\n"}});
|
||||
await assert.rejects(execFileAsync("bash",[extract,descriptorMismatch,join(run.root,"exports/extracted/descriptor-mismatch"),"p1-http"],{cwd:repo}),/workspace.*identity|descriptor/i);
|
||||
|
||||
const outside=join(repo,"outside-extract"); await mkdir(outside); await rm(join(run.root,"exports/extracted"),{recursive:true}); await symlink(outside,join(run.root,"exports/extracted"));
|
||||
const safe=await makeExportZip(run.root,"symlink-parent");
|
||||
await assert.rejects(execFileAsync("bash",[extract,safe,join(run.root,"exports/extracted/escape"),"p1-filesystem"],{cwd:repo}),/symlink|owned|unsafe/i);
|
||||
assert.deepEqual(await readdir(outside),[]); await rm(join(run.root,"exports/extracted")); await mkdir(join(run.root,"exports/extracted"));
|
||||
|
||||
const original=await makeExportZip(run.root,"replace-original"),replacement=await makeExportZip(run.root,"replace-malicious",{extra:true});
|
||||
const bin=join(run.root,"swap-bin"),markerPath=join(run.root,"swap-once"); await mkdir(bin);
|
||||
const realUnzip=(await execFileAsync("which",["unzip"])).stdout.trim();
|
||||
await writeFile(join(bin,"unzip"),`#!/bin/sh
|
||||
if [ ! -e "$P1_SWAP_MARKER" ]; then cp "$P1_SWAP_REPLACEMENT" "$P1_SWAP_ORIGINAL"; : > "$P1_SWAP_MARKER"; fi
|
||||
exec ${realUnzip} "$@"
|
||||
`,{mode:0o700});
|
||||
await execFileAsync("bash",[extract,original,join(run.root,"exports/extracted/staged-source"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:markerPath,P1_SWAP_REPLACEMENT:replacement,P1_SWAP_ORIGINAL:original}});
|
||||
await lstat(markerPath); await lstat(join(run.root,"exports/extracted/staged-source/manifest.json"));
|
||||
const generated=await readFile(extract,"utf8"); assert.match(generated,/source_fd = os\.open\(zip_path/); assert.match(generated,/dir_fd=base_fd/); assert.match(generated,/O_NOFOLLOW/); assert.match(generated,/staged archive SHA mismatch/);
|
||||
});
|
||||
|
||||
test("generated ZIP verifier anchors output when the extraction base is swapped on first unzip", async () => {
|
||||
const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh");
|
||||
const zip=await makeExportZip(run.root,"ancestor-swap"),base=join(run.root,"exports/extracted"),moved=join(run.root,"exports/extracted-original"),outside=join(repo,"outside-extraction-race");
|
||||
const bin=join(repo,"unzip-swap-bin"),marker=join(repo,"unzip-swapped"),realUnzip=(await execFileAsync("which",["unzip"])).stdout.trim();
|
||||
await mkdir(bin); await mkdir(outside);
|
||||
await writeFile(join(bin,"unzip"),`#!/bin/sh
|
||||
if [ ! -e "$P1_SWAP_MARKER" ]; then
|
||||
mv "$P1_SWAP_BASE" "$P1_SWAP_MOVED"
|
||||
ln -s "$P1_SWAP_OUTSIDE" "$P1_SWAP_BASE"
|
||||
: > "$P1_SWAP_MARKER"
|
||||
fi
|
||||
exec ${realUnzip} "$@"
|
||||
`,{mode:0o700});
|
||||
await assert.rejects(execFileAsync("bash",[extract,zip,join(base,"escaped"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:marker,P1_SWAP_BASE:base,P1_SWAP_MOVED:moved,P1_SWAP_OUTSIDE:outside}}),/owned extraction root|identity|changed|unsafe/i);
|
||||
await lstat(marker); assert.deepEqual(await readdir(outside),[]);
|
||||
});
|
||||
|
||||
test("generated ZIP verifier scans all four extracted byte streams for Evidence and canaries", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh");
|
||||
const markers=["P1 manually curated Evidence","DWH-"+"d".repeat(32),"CANARY-MUST-BE-REJECTED"];
|
||||
for(const marker of markers)for(const target of ["manifest.json","workspace.yaml","contract.env.example","README.md"]){
|
||||
const name=`scan-${markers.indexOf(marker)}-${target.replaceAll(".","-")}`,payloads=target==="manifest.json"?{}:{[target]:marker};
|
||||
const files={"workspace.yaml":"workspace:\n id: p1-filesystem\n","contract.env.example":"SAFE=path\n","README.md":"# Safe\n",...payloads};
|
||||
const manifest={schema_version:1,workspace_id:target==="manifest.json"?marker:"p1-filesystem",files:Object.fromEntries(Object.entries(files).map(([n,b])=>[n,sha256(b)]))};
|
||||
const zip=await makeExportZip(run.root,name,{payloads,manifest});
|
||||
await assert.rejects(execFileAsync("bash",[extract,zip,join(run.root,"exports/extracted",name),"p1-filesystem"],{cwd:repo}),error=>/Evidence|canary/.test(error.stderr)&&!error.stderr.includes(marker),`${target} must reject ${marker.slice(0,8)}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("generated secret scan excludes only the exact request fixture and hides fixed canary", async () => {
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const installed=join(run.root,"installation/registry/repo"),scan=join(run.root,"commands/secret-scan.sh"),canary="CANARY-MUST-BE-REJECTED";
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),installed]); await execFileAsync("bash",[scan],{cwd:repo});
|
||||
const leak=join(run.root,"responses/requests/invalid-credential.json"); await mkdir(dirname(leak),{recursive:true}); await writeFile(leak,canary);
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary));
|
||||
});
|
||||
|
||||
test("generated secret scan reads Git metadata and arbitrary dot-git directories without printing values", async () => {
|
||||
for(const rel of ["author/.git/manual-leak","responses/.git/leak"]){
|
||||
const canary="SECRET-"+"a".repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),scan=join(run.root,"commands/secret-scan.sh");
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]);
|
||||
await mkdir(dirname(join(run.root,rel)),{recursive:true}); await writeFile(join(run.root,rel),canary);
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary),`${rel} must be scanned with a redacted finding`);
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("generated secret scan reads raw dangling commit, tag, and tree objects without printing values", async () => {
|
||||
for(const kind of ["commit","tag","tree"]){
|
||||
const canary="SESSION-"+({commit:"b",tag:"c",tree:"d"}[kind]).repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),author=join(run.root,"author"),scan=join(run.root,"commands/secret-scan.sh");
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]);
|
||||
if(kind==="commit"){
|
||||
await execFileAsync("git",["commit","--allow-empty","-m",canary],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD^"],{cwd:author});
|
||||
}else if(kind==="tag"){
|
||||
await execFileAsync("git",["tag","-a","temporary-canary-tag","-m",canary],{cwd:author}); await execFileAsync("git",["tag","-d","temporary-canary-tag"],{cwd:author});
|
||||
}else{
|
||||
await writeFile(join(author,canary),"safe tree payload\n"); await execFileAsync("git",["add",canary],{cwd:author}); await execFileAsync("git",["write-tree"],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD"],{cwd:author});
|
||||
}
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git object/.test(error.stderr)&&!error.stderr.includes(canary),`${kind} raw bytes must be scanned with a redacted finding`);
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("generated secret scan checks unreachable blobs and dangling commits without printing values", async () => {
|
||||
for(const kind of ["unreachable-blob","dangling-commit"]){
|
||||
const value=kind==="unreachable-blob"?"SECRET-"+"e".repeat(32):"SECRET-"+"f".repeat(32);
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const author=join(run.root,"author"),installed=join(run.root,"installation/registry/repo"),scan=join(run.root,"commands/secret-scan.sh");
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),installed]); const file=join(author,"dangling-secret"); await writeFile(file,value);
|
||||
if(kind==="unreachable-blob"){await execFileAsync("git",["hash-object","-w",file],{cwd:author}); await rm(file);}
|
||||
else {await execFileAsync("git",["add","dangling-secret"],{cwd:author}); await execFileAsync("git",["commit","-m","dangling secret"],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD^"],{cwd:author});}
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git blob/.test(error.stderr)&&!error.stderr.includes(value),`${kind} must be scanned`);
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("generated secret scan checks randomized and fixed canaries in reachable Git without printing values", async () => {
|
||||
for(const canary of ["DWH-"+"c".repeat(32),"CANARY-MUST-BE-REJECTED"]){
|
||||
const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const installed=join(run.root,"installation/registry/repo"),author=join(run.root,"author"),scan=join(run.root,"commands/secret-scan.sh");
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),installed]);
|
||||
await writeFile(join(author,"temporary-secret"),canary); await execFileAsync("git",["add","temporary-secret"],{cwd:author}); await execFileAsync("git",["commit","-m","temporary canary"],{cwd:author}); await execFileAsync("git",["rm","temporary-secret"],{cwd:author}); await execFileAsync("git",["commit","-m","remove canary"],{cwd:author});
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git blob/.test(error.stderr)&&!error.stderr.includes(canary));
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("prepare publishes cleanable ownership before lab population", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim(),bin=join(repo,"failing-bin");
|
||||
await installFakeServer(repo); await mkdir(bin);
|
||||
await writeFile(join(bin,"git"),`#!/bin/sh
|
||||
if [ "$1" = init ]; then exit 71; fi
|
||||
exec ${JSON.stringify(realGit)} "$@"
|
||||
`,{mode:0o700});
|
||||
const prior=process.env.PATH; process.env.PATH=`${bin}:${prior}`;
|
||||
try { await assert.rejects(prepareManual({repositoryRoot:repo,skipBuild:true})); }
|
||||
finally { process.env.PATH=prior; }
|
||||
const owned=await readManualOwnership({repositoryRoot:repo});
|
||||
assert.equal(owned.stage,"PREPARING");
|
||||
await cleanupManual({repositoryRoot:repo});
|
||||
await assert.rejects(lstat(fixedManualRoot(repo)));
|
||||
});
|
||||
|
||||
test("stable repo-root lifecycle namespace survives manual-parent rename and cleans partial prepare", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim();
|
||||
const bin=join(repo,"rename-lock-bin"),entered=join(repo,"rename-entered"),release=join(repo,"rename-release");
|
||||
await installFakeServer(repo); await mkdir(bin);
|
||||
await writeFile(join(bin,"git"),`#!/bin/sh
|
||||
if [ "$1" = init ] && [ ! -e ${JSON.stringify(entered)} ]; then
|
||||
: > ${JSON.stringify(entered)}
|
||||
while [ ! -e ${JSON.stringify(release)} ]; do sleep 0.01; done
|
||||
fi
|
||||
exec ${JSON.stringify(realGit)} "$@"
|
||||
`,{mode:0o700});
|
||||
const prior=process.env.PATH; process.env.PATH=`${bin}:${prior}`; let preparing;
|
||||
const parent=join(repo,".artifacts/manual-acceptance"),moved=join(repo,".artifacts/manual-acceptance-moved");
|
||||
try {
|
||||
preparing=prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
for(let n=0;n<300;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,10));}
|
||||
await lstat(entered); await rename(parent,moved); await mkdir(parent,{recursive:true}); await writeFile(join(parent,"public-sibling"),"keep"); await writeFile(join(moved,"moved-sibling"),"keep");
|
||||
await assert.rejects(cleanupManual({repositoryRoot:repo}),/lifecycle lock|operator inspection/i);
|
||||
await writeFile(release,"go"); await assert.rejects(preparing,/identity|changed|unsafe|manual/i); preparing=undefined;
|
||||
assert.equal(await readFile(join(parent,"public-sibling"),"utf8"),"keep");
|
||||
assert.equal(await readFile(join(moved,"moved-sibling"),"utf8"),"keep");
|
||||
await assert.rejects(lstat(join(moved,"p1")));
|
||||
await assert.rejects(lstat(join(repo,".p1-manual-acceptance.lifecycle.lock")));
|
||||
} finally { process.env.PATH=prior; await writeFile(release,"go").catch(()=>{}); if(preparing)await preparing.catch(()=>{}); }
|
||||
});
|
||||
|
||||
test("all four lifecycle operations serialize on the stable repo-root lock", async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); await prepareManual({repositoryRoot:repo,skipBuild:true});
|
||||
const lock=join(repo,".p1-manual-acceptance.lifecycle.lock"); await writeFile(lock,"foreign",{mode:0o600});
|
||||
try {
|
||||
for(const operation of [prepareManual,serveManual,stopManual,cleanupManual])
|
||||
await assert.rejects(operation({repositoryRoot:repo,skipBuild:true}),/lifecycle lock|operator inspection/i);
|
||||
} finally { await rm(lock,{force:true}); }
|
||||
});
|
||||
|
||||
test("prepare binds production entry bytes and serve rejects a regular replacement", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),malicious=join(repo,"malicious-executed"); await installFakeServer(repo);
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo}),script=join(repo,"backend/dist/server.js");
|
||||
const prepared=await readFile(script); assert.equal(owned.entrypoint.sha256,sha256(prepared));
|
||||
await rm(script); await writeFile(script,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(malicious)},"bad");setInterval(()=>{},1000);`);
|
||||
await assert.rejects(serveManual({repositoryRoot:repo}),/entrypoint|production server.*identity/i);
|
||||
await assert.rejects(lstat(malicious)); await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
});
|
||||
|
||||
test("serve rejects replacement of an imported production dependency", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),malicious=join(repo,"dependency-executed"); await installFakeServer(repo);
|
||||
const server=join(repo,"backend/dist/server.js"), original=await readFile(server); await writeFile(join(repo,"backend/dist/dep.js"),"export const dependency = true;\n"); await writeFile(server,`import \"./dep.js\";\n${original}`);
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const before=await readFile(server); await writeFile(join(repo,"backend/dist/dep.js"),`import {writeFileSync} from \"node:fs\"; writeFileSync(${JSON.stringify(malicious)},\"bad\");\n`);
|
||||
assert.deepEqual(await readFile(server),before); await assert.rejects(serveManual({repositoryRoot:repo}),/distribution|identity|manifest/i); await assert.rejects(lstat(malicious)); await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
});
|
||||
|
||||
test("serve refuses a replaced backend dist dependency after prepare", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo);
|
||||
const dependency=join(repo,"backend/dist/dependency.js"); await writeFile(dependency,"export const value = 1;\n");
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),marker=join(repo,"dependency-replaced-executed");
|
||||
await writeFile(dependency,`import { writeFileSync } from "node:fs";writeFileSync(${JSON.stringify(marker)},"bad");export const value = 2;\n`);
|
||||
await assert.rejects(serveManual({repositoryRoot:repo}),/distribution|manifest|identity/i);
|
||||
await assert.rejects(lstat(marker)); await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
});
|
||||
|
||||
test("serve refuses a deterministic dependency check/load swap before execution", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),marker=join(repo,"dep-swap-executed");
|
||||
await writeFile(join(repo,"backend/dist/dep.js"),`export function start(){}\n`);
|
||||
await writeFile(join(repo,"backend/dist/server.js"),`import http from "node:http";\nimport { start } from "./dep.js";\nstart();\nconst server=http.createServer((req,res)=>{res.statusCode=req.url==="/health"?200:200;res.setHeader("content-type","application/json");res.end(JSON.stringify({status:"ok"}));});\nserver.listen(Number(process.env.PORT),process.env.HOST);\n`);
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo}),replacement=join(repo,"dep-replacement.js");
|
||||
await writeFile(replacement,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(marker)},"executed");export function start(){}\n`);
|
||||
await assert.rejects(serveManual({repositoryRoot:repo,beforeSpawn:async()=>rename(replacement,join(repo,"backend/dist/dep.js"))}),/identity|changed|refused|distribution|module|readiness|failed/i);
|
||||
await assert.rejects(lstat(marker)); await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); assert.deepEqual(await listenerPids(),[]);
|
||||
});
|
||||
|
||||
test("immutable loader serves verified cached dependency bytes after a same-path regular replacement", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),marker=join(repo,"dep-replacement-executed");
|
||||
await writeFile(join(repo,"backend/dist/dep.js"),`export function mark(){ globalThis.__depSource = "original"; }\n`);
|
||||
await writeFile(join(repo,"backend/dist/server.js"),`import http from "node:http";\nlet n = 0;\nconst server=http.createServer(async (req,res)=>{ if(req.url==="/load"){ await import(\`./dep.js?v=\${++n}\`).then(m=>m.mark()); } res.setHeader("content-type","application/json"); res.end(JSON.stringify({status:"ok",dep:globalThis.__depSource??"unset"})); });\nserver.listen(Number(process.env.PORT),process.env.HOST);\n`);
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo});
|
||||
const pid=await serveManual({repositoryRoot:repo});
|
||||
try {
|
||||
const health=async()=>(await (await fetch("http://127.0.0.1:8791/health")).json());
|
||||
const load=async()=>{ await fetch("http://127.0.0.1:8791/load"); return (await health()).dep; };
|
||||
for(let n=0;n<60;n++){try{if((await health()).status==="ok")break;}catch{}await new Promise(r=>setTimeout(r,50));}
|
||||
assert.equal(await load(),"original");
|
||||
await writeFile(join(repo,"backend/dist/dep.js"),`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(marker)},"executed");export function mark(){ globalThis.__depSource = "replaced"; }\n`);
|
||||
assert.equal(await load(),"original"); await assert.rejects(lstat(marker));
|
||||
await writeFile(join(repo,"backend/dist/dep.js"),`export function mark(){ globalThis.__depSource = "original"; }\n`);
|
||||
} finally {
|
||||
try { await stopManual({repositoryRoot:repo}); } catch { try { process.kill(pid,"SIGTERM"); } catch {} }
|
||||
}
|
||||
await cleanupManual({repositoryRoot:repo});
|
||||
});
|
||||
|
||||
test("opened production FD prevents deterministic check-spawn replacement execution", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),safe=join(repo,"safe-executed"),malicious=join(repo,"malicious-executed"); await installFakeServer(repo,{marker:safe});
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),script=join(repo,"backend/dist/server.js"),replacement=join(repo,"replacement-server.js"),owned=await readManualOwnership({repositoryRoot:repo});
|
||||
await writeFile(replacement,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(malicious)},"bad");setInterval(()=>{},1000);`);
|
||||
await assert.rejects(serveManual({repositoryRoot:repo,beforeSpawn:async()=>rename(replacement,script)}),/entrypoint|identity|changed|readiness|failed|distribution|module/i);
|
||||
await assert.rejects(lstat(malicious)); await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]);
|
||||
});
|
||||
|
||||
test("foreign 8791 health can never make a delayed authenticated child RUNNING", { concurrency: false }, async () => {
|
||||
const repo=await fakeRepo(),entered=join(repo,"entry-loaded"); await installFakeServer(repo,{startupDelay:700,marker:entered});
|
||||
const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo});
|
||||
const serving=serveManual({repositoryRoot:repo});
|
||||
for(let n=0;n<300;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,5));}
|
||||
await lstat(entered); const foreign=net.createServer((socket)=>socket.end("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nforeign"));
|
||||
await new Promise((resolvePromise,reject)=>foreign.once("error",reject).listen(8791,"127.0.0.1",resolvePromise));
|
||||
try {
|
||||
await assert.rejects(serving,/readiness|listener|entrypoint|backend failed/i);
|
||||
await assert.rejects(lstat(join(run.root,"backend.pid")));
|
||||
assert.equal((await listenerPids()).includes(process.pid),true);
|
||||
assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]);
|
||||
} finally { await new Promise(resolvePromise=>foreign.close(resolvePromise)); }
|
||||
});
|
||||
|
||||
test("absence gate rejects evidence and every P2 materialization artifact name", async () => {
|
||||
for(const rel of ["artifacts/evidence/generation/chunk.md","responses/materialization","responses/preprocess-state","responses/embedding-cache","responses/qdrant-state","responses/ACTIVE","responses/retention-policy"]){
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),check=join(run.root,"commands/absence-check.sh"),target=join(run.root,rel);
|
||||
if(rel.endsWith("materialization"))await mkdir(target,{recursive:true}); else {await mkdir(dirname(target),{recursive:true}); await writeFile(target,"safe");}
|
||||
await assert.rejects(execFileAsync("bash",[check],{cwd:repo}),/out-of-scope/i,rel);
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("secret scan discovers every arbitrary git repository including unreachable objects", async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),gitdir=join(run.root,"responses/.git"),scan=join(run.root,"commands/secret-scan.sh"),canary="SECRET-"+"9".repeat(32);
|
||||
await execFileAsync("git",["init","--bare",gitdir]); const blob=join(run.root,"responses/canary-blob"); await writeFile(blob,canary); await execFileAsync("git",["--git-dir",gitdir,"hash-object","-w",blob]); await rm(blob);
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git object|secret canary/.test(error.stderr)&&!error.stderr.includes(canary));
|
||||
});
|
||||
|
||||
test("secret scan bounds and scans filesystem names plus loose ref names", async () => {
|
||||
for(const kind of ["file","directory","loose-ref"]){
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),scan=join(run.root,"commands/secret-scan.sh"),canary="SESSION-"+"8".repeat(32);
|
||||
if(kind==="file")await writeFile(join(run.root,"responses",canary),"safe");
|
||||
if(kind==="directory")await mkdir(join(run.root,"responses",canary));
|
||||
if(kind==="loose-ref"){const ref=join(run.root,"author/.git/refs/heads",canary);await mkdir(dirname(ref),{recursive:true});await writeFile(ref,"0".repeat(40)+"\n");}
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary/.test(error.stderr)&&!error.stderr.includes(canary),kind);
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("extractor and scanner operational diagnostics redact canary-bearing paths", async () => {
|
||||
const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),canary="SECRET-"+"7".repeat(32),extract=join(run.root,"commands/extract-export.sh"),scan=join(run.root,"commands/secret-scan.sh");
|
||||
const missing=join(run.root,"exports/raw",`${canary}.zip`),output=join(run.root,"exports/extracted",canary);
|
||||
await assert.rejects(execFileAsync("bash",[extract,missing,output,"p1-filesystem"],{cwd:repo}),error=>!error.stderr.includes(canary)&&/redacted|refused|unsafe/i.test(error.stderr));
|
||||
const oversized=join(run.root,"responses","oversized"); const handle=await open(oversized,"w"); await handle.truncate(33554433); await handle.close();
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>!error.stderr.includes(canary)&&/redacted|failed|bound/i.test(error.stderr));
|
||||
});
|
||||
|
||||
|
||||
test("public prepare build is inside the stable lifecycle transaction", async()=>{
|
||||
const wrapper=await readFile(new URL("../../scripts/p1-manual-acceptance.sh",import.meta.url),"utf8"),source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8");
|
||||
assert.doesNotMatch(wrapper,/npm .*run build/); assert.match(source,/action==="prepare"\)await prepareManual\(\)/);
|
||||
});
|
||||
@@ -1,172 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants, lstatSync, realpathSync } from "node:fs";
|
||||
import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
// This acceptance-only adapter deliberately imports the built production runner.
|
||||
import { ThtRunner } from "../dist/tht/tht-runner.js";
|
||||
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const HEX40 = /^[0-9a-f]{40}$/;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
|
||||
function fixedRoot(repositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p1"); }
|
||||
function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); }
|
||||
function assertNoSymlinks(root, path, allowMissingLeaf = false) {
|
||||
const rel = relative(root, path);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path is outside owned root");
|
||||
let cursor = root;
|
||||
for (const [index, part] of rel.split(sep).filter(Boolean).entries()) {
|
||||
cursor = join(cursor, part);
|
||||
try { if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink"); }
|
||||
catch (error) {
|
||||
if (allowMissingLeaf && error.code === "ENOENT" && index === rel.split(sep).filter(Boolean).length - 1) return;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
async function ownership(repositoryRoot, ownershipPath) {
|
||||
const root = fixedRoot(repositoryRoot);
|
||||
const expected = join(root, "ownership.json");
|
||||
if (resolve(ownershipPath) !== expected) throw new Error("ownership path is not owned");
|
||||
const rootEntry = await lstat(root); const ownershipEntry = await lstat(expected);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership is unsafe");
|
||||
if (await realpath(root) !== root) throw new Error("ownership root is not canonical");
|
||||
let value; try { value = JSON.parse(await readFile(expected, "utf8")); } catch { throw new Error("ownership is malformed"); }
|
||||
if (value?.schemaVersion !== 1 || value.kind !== "p1-manual-acceptance" || !HEX64.test(value.nonce ?? "")
|
||||
|| value.repositoryRoot !== realpathSync(repositoryRoot) || value.root !== root || value.status !== "PENDING"
|
||||
|| value.listener?.host !== "127.0.0.1" || value.listener?.port !== 8791) throw new Error("ownership identity mismatch");
|
||||
return { root, value };
|
||||
}
|
||||
const ANCHORED_PUBLISH_SOURCE=String.raw`import os,secrets,stat,sys
|
||||
parent,name,expected_dev,expected_ino=sys.argv[1:]
|
||||
pfd=fd=None;stage=".render-stage-"+secrets.token_hex(16);published=False
|
||||
def fail(): raise RuntimeError("anchored publication refused")
|
||||
try:
|
||||
pfd=os.open(parent,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
||||
identity=os.fstat(pfd)
|
||||
if (identity.st_dev,identity.st_ino)!=(int(expected_dev),int(expected_ino)): fail()
|
||||
try: os.stat(name,dir_fd=pfd,follow_symlinks=False); fail()
|
||||
except FileNotFoundError: pass
|
||||
fd=os.open(stage,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=pfd)
|
||||
data=sys.stdin.buffer.read(33554433)
|
||||
if len(data)>33554432: fail()
|
||||
view=memoryview(data)
|
||||
while view:
|
||||
written=os.write(fd,view)
|
||||
if written<=0: fail()
|
||||
view=view[written:]
|
||||
os.fsync(fd);os.close(fd);fd=None;os.rename(stage,name,src_dir_fd=pfd,dst_dir_fd=pfd);published=True;os.fsync(pfd)
|
||||
current=os.stat(parent,follow_symlinks=False)
|
||||
if not stat.S_ISDIR(current.st_mode) or (current.st_dev,current.st_ino)!=(identity.st_dev,identity.st_ino): fail()
|
||||
except Exception:
|
||||
if published:
|
||||
try: os.unlink(name,dir_fd=pfd);os.fsync(pfd)
|
||||
except Exception: pass
|
||||
print("anchored output publication refused (details redacted)",file=sys.stderr);raise SystemExit(1)
|
||||
finally:
|
||||
if fd is not None: os.close(fd)
|
||||
if pfd is not None:
|
||||
try: os.unlink(stage,dir_fd=pfd)
|
||||
except FileNotFoundError: pass
|
||||
os.close(pfd)
|
||||
`;
|
||||
async function atomicCopy(source,output) {
|
||||
const parent=dirname(output),entry=await lstat(parent);if(!entry.isDirectory()||entry.isSymbolicLink())throw new Error("rendered parent identity is unsafe");const bytes=await readFile(source);
|
||||
const result=spawnSync("python3",["-c",ANCHORED_PUBLISH_SOURCE,parent,basename(output),String(entry.dev),String(entry.ino)],{input:bytes,encoding:"utf8",maxBuffer:1024*1024});
|
||||
if(result.error||result.status!==0)throw new Error("anchored output publication refused; rendered parent identity changed or output is unsafe");
|
||||
}
|
||||
|
||||
function sameEntry(actual, expected) { return actual.dev === expected.dev && actual.ino === expected.ino; }
|
||||
async function readBounded(path, max, label) {
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const before = await handle.stat(), pathEntry = await lstat(path);
|
||||
if (!before.isFile() || pathEntry.isSymbolicLink() || !pathEntry.isFile() || !sameEntry(before, pathEntry)) throw new Error(`${label} is unsafe`);
|
||||
if (before.size < 1 || before.size > max) throw new Error(`${label} is unbounded`);
|
||||
const bytes = Buffer.alloc(before.size); let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset);
|
||||
if (bytesRead < 1) throw new Error(`${label} changed while reading`);
|
||||
offset += bytesRead;
|
||||
}
|
||||
const after = await handle.stat();
|
||||
if (!sameEntry(before, after) || after.size !== before.size) throw new Error(`${label} changed while reading`);
|
||||
return bytes;
|
||||
} finally {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
async function readSnapshotManifest(root, manifestPath, commit, yamlName, expectedDigest) {
|
||||
let manifestEntry;
|
||||
try { assertNoSymlinks(root, manifestPath); manifestEntry = await lstat(manifestPath); }
|
||||
catch (error) { if (error?.code === "ENOENT") throw new Error("snapshot manifest is missing or unbounded"); throw error; }
|
||||
if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink() || await realpath(manifestPath) !== manifestPath) throw new Error("snapshot manifest is unsafe");
|
||||
const bytes = await readBounded(manifestPath, 1048576, "snapshot manifest");
|
||||
let manifest; try { manifest = JSON.parse(bytes.toString("utf8")); } catch { throw new Error("snapshot manifest is malformed"); }
|
||||
const files = manifest?.files;
|
||||
if (manifest?.head !== commit || !files || typeof files !== "object" || Array.isArray(files)) throw new Error("snapshot manifest identity is unsafe");
|
||||
if (!HEX64.test(files[yamlName] ?? "") || files[yamlName] !== expectedDigest) throw new Error("snapshot manifest digest is unsafe");
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export async function renderOwnedSnapshot({ repositoryRoot = defaultRepositoryRoot, ownershipPath, snapshotPath, outputPath, snapshotSha256, env = process.env, beforePublish }) {
|
||||
const repo = realpathSync(repositoryRoot); const { root } = await ownership(repo, resolve(repo, ownershipPath));
|
||||
const snapshot = resolve(repo, snapshotPath); const output = resolve(repo, outputPath);
|
||||
const snapshotsRoot = join(root, "installation", "registry", "snapshots");
|
||||
const renderedRoot = join(root, "rendered");
|
||||
if (!isAbsolute(snapshotPath) || !below(snapshotsRoot, snapshot)) throw new Error("snapshot is not an owned absolute path");
|
||||
const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(relative(snapshotsRoot, snapshot).split(sep).join("/"));
|
||||
if (!match || !HEX40.test(match[1])) throw new Error("snapshot is not commit addressed");
|
||||
if (!HEX64.test(snapshotSha256 ?? "")) throw new Error("snapshot digest identity is unsafe");
|
||||
assertNoSymlinks(root, snapshot); const snapshotEntry = await lstat(snapshot);
|
||||
if (!snapshotEntry.isFile() || snapshotEntry.isSymbolicLink() || await realpath(snapshot) !== snapshot) throw new Error("snapshot is unsafe");
|
||||
const yamlName = `${match[2]}.yaml`;
|
||||
const manifestPath = join(snapshotsRoot, match[1], "snapshot.json");
|
||||
await readSnapshotManifest(root, manifestPath, match[1], yamlName, snapshotSha256);
|
||||
const snapshotBytes = await readBounded(snapshot, 1048576, "snapshot");
|
||||
if (createHash("sha256").update(snapshotBytes).digest("hex") !== snapshotSha256) throw new Error("snapshot bytes changed");
|
||||
if (!below(renderedRoot, output) || dirname(output) !== renderedRoot || !output.endsWith(".yaml")) throw new Error("output is not an owned rendered path");
|
||||
assertNoSymlinks(root, dirname(output));
|
||||
try { if ((await lstat(output)).isSymbolicLink()) throw new Error("output is unsafe"); } catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
await mkdir(join(snapshotsRoot, "runtime"), { recursive: true, mode: 0o700 });
|
||||
const prior = {};
|
||||
for (const [key, value] of Object.entries(env)) { prior[key] = process.env[key]; if (value === undefined) delete process.env[key]; else process.env[key] = value; }
|
||||
const runner = new ThtRunner({
|
||||
thtBin: join(repo, "harness", ".venv", "bin", "tht"), harnessDir: join(repo, "harness"),
|
||||
configPath: join(root, "installation", "base.yaml"), dataRoot: join(root, "installation", "data"),
|
||||
runtimeSnapshotRoot: join(snapshotsRoot, "runtime"), secretRoots: [join(root, "fixture-secrets")],
|
||||
semanticRuntime: { internalQdrantUrl: "http://qdrant:6333", internalEmbeddingUrl: "http://embedding:11434", internalEmbeddingModel: "qwen3-embedding:0.6b", internalEmbeddingDimensions: 1024 },
|
||||
});
|
||||
let lease;
|
||||
try {
|
||||
lease = runner.acquireWorkspaceRuntime(snapshot);
|
||||
const verifySnapshot = async () => {
|
||||
const current = await readBounded(snapshot, 1048576, "snapshot");
|
||||
if (createHash("sha256").update(current).digest("hex") !== snapshotSha256) throw new Error("snapshot content changed during rendering");
|
||||
};
|
||||
await verifySnapshot();
|
||||
if(beforePublish)await beforePublish({output,renderedRoot});
|
||||
await verifySnapshot();
|
||||
await atomicCopy(lease.path, output);
|
||||
}
|
||||
finally {
|
||||
if (lease) lease.release();
|
||||
for (const key of Object.keys(env)) { if (prior[key] === undefined) delete process.env[key]; else process.env[key] = prior[key]; }
|
||||
}
|
||||
return output;
|
||||
}
|
||||
function parseArgs(argv) {
|
||||
if (argv.length !== 8) throw new Error("usage: p1-render-snapshot.mjs --ownership PATH --snapshot ABSOLUTE_PATH --output PATH --snapshot-sha256 HEX");
|
||||
const result = {}; for (let i=0;i<argv.length;i+=2) { if (!["--ownership","--snapshot","--output","--snapshot-sha256"].includes(argv[i]) || result[argv[i]]) throw new Error("invalid arguments"); result[argv[i]]=argv[i+1]; }
|
||||
if (!result["--ownership"] || !result["--snapshot"] || !result["--output"] || !result["--snapshot-sha256"]) throw new Error("missing arguments"); return result;
|
||||
}
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) {
|
||||
try { const args=parseArgs(process.argv.slice(2)); await renderOwnedSnapshot({ ownershipPath:args["--ownership"], snapshotPath:args["--snapshot"], outputPath:args["--output"], snapshotSha256:args["--snapshot-sha256"] }); console.log(`rendered ${resolve(args["--output"])}`); }
|
||||
catch(error) { console.error(`p1 render refused: ${error.message}`); process.exitCode=1; }
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { renderOwnedSnapshot } from "./p1-render-snapshot.mjs";
|
||||
|
||||
const roots=[];
|
||||
const sha256=bytes=>createHash("sha256").update(bytes).digest("hex");
|
||||
async function fixture() {
|
||||
const repo=await realpath(await mkdtemp(join(tmpdir(),"p1-render-repo-"))); roots.push(repo);
|
||||
const root=join(repo,".artifacts/manual-acceptance/p1"); const commit="a".repeat(40); const snapshot=join(root,"installation/registry/snapshots",commit,"p1-filesystem.yaml");
|
||||
for (const p of [dirname(snapshot),join(root,"rendered"),join(root,"installation/registry/snapshots/runtime"),join(root,"installation/data"),join(root,"fixture-secrets"),join(repo,"harness")]) await mkdir(p,{recursive:true,mode:0o700});
|
||||
await writeFile(join(root,"ownership.json"),JSON.stringify({schemaVersion:1,kind:"p1-manual-acceptance",nonce:"b".repeat(64),repositoryRoot:repo,root,status:"PENDING",listener:{host:"127.0.0.1",port:8791}}));
|
||||
await writeFile(join(root,"installation/base.yaml"),"{}\n");
|
||||
const secret=join(root,"fixture-secrets/dwh-password"); await writeFile(secret,"not-inspected",{mode:0o600});
|
||||
await writeFile(snapshot,`workspace:
|
||||
schema_version: 3
|
||||
id: p1-filesystem
|
||||
name: P1 filesystem
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: public
|
||||
supported_transports: [postgres_direct]
|
||||
semantic_index:
|
||||
vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
|
||||
embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
|
||||
llm_policy:
|
||||
allowed: [zai/glm-5.2]
|
||||
evidence:
|
||||
source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
|
||||
policy: {max_chunk_chars: 4000, retain_published_generations: 3}
|
||||
`);
|
||||
const snapshotBytes=await readFile(snapshot); const snapshotSha256=sha256(snapshotBytes);
|
||||
const manifestPath=join(dirname(snapshot),"snapshot.json");
|
||||
await writeFile(manifestPath,JSON.stringify({head:commit,revisions:[{id:"p1-filesystem",commit,blob:"0".repeat(40),snapshotPath:snapshot}],files:{"p1-filesystem.yaml":snapshotSha256}}));
|
||||
const env={THT_WS_P1_FILESYSTEM_DWH_TRANSPORT:"postgres_direct",THT_WS_P1_FILESYSTEM_DWH_HOST:"dwh.invalid",THT_WS_P1_FILESYSTEM_DWH_PORT:"5432",THT_WS_P1_FILESYSTEM_DWH_USER:"reader",THT_WS_P1_FILESYSTEM_DWH_PASSWORD_FILE:secret};
|
||||
return {repo,root,snapshot,snapshotSha256,manifestPath,env};
|
||||
}
|
||||
test.afterEach(async()=>Promise.all(roots.splice(0).map(r=>rm(r,{recursive:true,force:true}))));
|
||||
|
||||
const call=(f,extra={})=>renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:f.snapshot,snapshotSha256:f.snapshotSha256,env:{...process.env,...f.env},...extra});
|
||||
const runtimeLeases=async f=>await (await import("node:fs/promises")).readdir(join(f.root,"installation/registry/snapshots/runtime"));
|
||||
|
||||
test("renderer copies a production lease deterministically with mode 0600 and no leases",async()=>{ const f=await fixture(); const one=join(f.root,"rendered/one.yaml"),two=join(f.root,"rendered/two.yaml"); await call(f,{outputPath:one}); await call(f,{outputPath:two}); assert.deepEqual(await readFile(one),await readFile(two)); assert.equal((await lstat(one)).mode&0o777,0o600); assert.deepEqual(await runtimeLeases(f),[]); });
|
||||
|
||||
test("renderer rejects unowned, symlink, out-of-root and missing-digest paths",async()=>{ const f=await fixture(); const outside=join(f.repo,"outside.yaml"); await writeFile(outside,"x"); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:outside,outputPath:join(f.root,"rendered/x.yaml"),snapshotSha256:f.snapshotSha256,env:f.env}),/owned|snapshot/); const link=join(dirname(f.snapshot),"linked.yaml"); await symlink(f.snapshot,link); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:link,outputPath:join(f.root,"rendered/x.yaml"),snapshotSha256:f.snapshotSha256,env:f.env}),/snapshot|symlink/); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:f.snapshot,outputPath:outside,snapshotSha256:f.snapshotSha256,env:f.env}),/output/); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:f.snapshot,outputPath:join(f.root,"rendered/x.yaml"),env:f.env}),/snapshot digest identity/); });
|
||||
|
||||
test("renderer releases its acquired lease when atomic output copy fails",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/existing.yaml"); await mkdir(output); await assert.rejects(call(f,{outputPath:output}),/anchored|publication|unsafe/); assert.deepEqual(await runtimeLeases(f),[]); });
|
||||
|
||||
test("renderer refuses a same-path regular snapshot byte replacement against manifest and expected digest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); const replaced=(await readFile(f.snapshot,"utf8")).replace("max_chunk_chars: 4000","max_chunk_chars: 3999"); await writeFile(f.snapshot,replaced); await assert.rejects(call(f,{outputPath:output}),/snapshot bytes changed/); await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); });
|
||||
|
||||
test("renderer refuses snapshot manifest head, digest, and expected-digest tampering",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/tampered.yaml"); const manifest=JSON.parse(await readFile(f.manifestPath,"utf8"));
|
||||
await writeFile(f.manifestPath,JSON.stringify({...manifest,head:"c".repeat(40)})); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest identity/);
|
||||
await writeFile(f.manifestPath,JSON.stringify({...manifest,files:{"p1-filesystem.yaml":"d".repeat(64)}})); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest digest/);
|
||||
await writeFile(f.manifestPath,JSON.stringify(manifest)); await assert.rejects(call(f,{outputPath:output,snapshotSha256:"e".repeat(64)}),/snapshot manifest digest/);
|
||||
await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); });
|
||||
|
||||
test("renderer refuses a missing or malformed snapshot manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/nomanifest.yaml"); await rm(f.manifestPath); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*(missing|unbounded|unsafe)/); await writeFile(f.manifestPath,"{not json"); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*malformed/); await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); });
|
||||
|
||||
test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n schema_version: 3\n id: p1-filesystem\n name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); });
|
||||
|
||||
test("renderer anchors publication when rendered parent is concurrently swapped", async()=>{
|
||||
const f=await fixture(),output=join(f.root,"rendered/raced.yaml"),moved=join(f.root,"rendered-moved"),outside=join(f.repo,"outside-rendered"); await mkdir(outside);
|
||||
await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await rename(join(f.root,"rendered"),moved);await symlink(outside,join(f.root,"rendered"));}}),/identity|changed|unsafe|publication/i);
|
||||
assert.deepEqual(await (await import("node:fs/promises")).readdir(outside),[]);
|
||||
});
|
||||
@@ -1,905 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { access, lstat, mkdir, open, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
buildSafeEnvironment,
|
||||
collectRepositoryProvenance,
|
||||
deriveOverall,
|
||||
scanSecrets,
|
||||
} from "./p1-acceptance.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const RUN_ID = /^p11-[0-9a-f]{32}$/;
|
||||
const HEX40 = /^[0-9a-f]{40}$/;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
||||
const ZIP_FILES = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"];
|
||||
|
||||
function resolveSystemExecutable(name) {
|
||||
for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) {
|
||||
try {
|
||||
const resolved = realpathSync(candidate);
|
||||
if (lstatSync(resolved).isFile()) return resolved;
|
||||
} catch {}
|
||||
}
|
||||
throw new Error(`required executable not found: ${name}`);
|
||||
}
|
||||
function resolveExecutables(repositoryRoot) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const thtPath = join(repo, "harness", ".venv", "bin", "tht");
|
||||
if (!existsSync(thtPath)) throw new Error("required executable not found: tht");
|
||||
return { gitPath: resolveSystemExecutable("git"), pythonPath: resolveSystemExecutable("python3"), thtPath: realpathSync(thtPath) };
|
||||
}
|
||||
const TOPOLOGY = [
|
||||
"remote.git", "author", "installation/registry", "installation/data", "installation/runtime",
|
||||
"fixture-secrets", "fixtures/descriptors", "fixtures/requests", "requests", "responses",
|
||||
"exports/raw", "exports/extracted", "rendered", "logs",
|
||||
];
|
||||
export const CHECK_IDS = Object.freeze([
|
||||
"preflight",
|
||||
"clean_state",
|
||||
"ownership",
|
||||
"catalog_bootstrap",
|
||||
"catalog_only_listing",
|
||||
"bootstrap_create_once",
|
||||
"api_curator_boundary",
|
||||
"curator_descriptor_update",
|
||||
"content_only_revision",
|
||||
"docs_only_reconciliation",
|
||||
"same_revision_git_objects",
|
||||
"snapshot_and_export",
|
||||
"runtime_render_determinism",
|
||||
"tht_config_check",
|
||||
"negative_catalog_layout_cases",
|
||||
"negative_schema_context_cases",
|
||||
"no_p2_scope_artifacts",
|
||||
"secret_scan",
|
||||
"cleanup_confinement",
|
||||
]);
|
||||
|
||||
function nowIso() { return new Date().toISOString(); }
|
||||
function sha256(value) { return createHash("sha256").update(value).digest("hex"); }
|
||||
function assert(condition, message) { if (!condition) throw new Error(message); }
|
||||
function scalarSecretBytes(value) {
|
||||
if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid");
|
||||
return Buffer.from(value);
|
||||
}
|
||||
function canonicalRoot(repositoryRoot) { return realpathSync(repositoryRoot); }
|
||||
export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) {
|
||||
return join(canonicalRoot(repositoryRoot), ".artifacts", "p11-integration");
|
||||
}
|
||||
export function validateRunRoot(repositoryRoot, runRoot, runId) {
|
||||
if (!RUN_ID.test(runId)) throw new Error("invalid owned run id");
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const lexical = resolve(runRoot);
|
||||
if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child");
|
||||
return lexical;
|
||||
}
|
||||
function validateNoSymlinkAncestors(repositoryRoot, target) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const rel = relative(repo, target);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository");
|
||||
let cursor = repo;
|
||||
for (const part of rel.split(sep).filter(Boolean)) {
|
||||
cursor = join(cursor, part);
|
||||
if (!existsSync(cursor)) break;
|
||||
const entry = lstatSync(cursor);
|
||||
if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink");
|
||||
}
|
||||
}
|
||||
async function atomicWrite(path, bytes, mode = 0o600) {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`);
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(staging, "wx", mode);
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
await rename(staging, path);
|
||||
const directory = openSync(dirname(path), fsConstants.O_RDONLY);
|
||||
try { fsyncSync(directory); } finally { closeSync(directory); }
|
||||
} catch (error) {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
await rm(staging, { force: true }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function exactOwnedResources(run) {
|
||||
return [
|
||||
run.root,
|
||||
join(run.root, "remote.git"),
|
||||
join(run.root, "author"),
|
||||
join(run.root, "installation", "registry"),
|
||||
join(run.root, "installation", "data"),
|
||||
join(run.root, "installation", "runtime"),
|
||||
];
|
||||
}
|
||||
function initialListeners(pid) {
|
||||
return [{ name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0, pid, state: "not_started" }];
|
||||
}
|
||||
function ownershipValue(run, listeners = run.listeners) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "p11-acceptance",
|
||||
runId: run.runId,
|
||||
runNonce: run.nonce,
|
||||
root: run.root,
|
||||
repositoryRoot: run.repositoryRoot,
|
||||
startedAt: run.startedAt,
|
||||
pid: run.pid,
|
||||
listeners,
|
||||
resources: exactOwnedResources(run),
|
||||
};
|
||||
}
|
||||
async function writeOwnership(run, listenerUpdate) {
|
||||
const listeners = listenerUpdate
|
||||
? run.listeners.map((listener) => listener.name === listenerUpdate.name ? listenerUpdate : listener)
|
||||
: run.listeners;
|
||||
await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run, listeners), null, 2)}\n`);
|
||||
run.listeners = listeners;
|
||||
}
|
||||
export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const base = canonicalIntegrationBase(repo);
|
||||
validateNoSymlinkAncestors(repo, base);
|
||||
await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; });
|
||||
await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; });
|
||||
const id = runId ?? `p11-${randomBytes(16).toString("hex")}`;
|
||||
const root = validateRunRoot(repo, join(base, id), id);
|
||||
const run = {
|
||||
repositoryRoot: repo,
|
||||
root,
|
||||
runId: id,
|
||||
nonce: nonce ?? randomBytes(32).toString("hex"),
|
||||
startedAt: now ?? nowIso(),
|
||||
pid: pid ?? process.pid,
|
||||
listeners: initialListeners(pid ?? process.pid),
|
||||
};
|
||||
if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity");
|
||||
await mkdir(root, { mode: 0o700 });
|
||||
await writeOwnership(run);
|
||||
return run;
|
||||
}
|
||||
function strictOwnership(value, run, expectedNonce) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed");
|
||||
const listener = value.listeners?.[0];
|
||||
const validListener = Array.isArray(value.listeners) && value.listeners.length === 1
|
||||
&& listener?.name === "primary" && listener.kind === "fastify" && listener.host === "127.0.0.1"
|
||||
&& listener.requestedPort === 0 && listener.pid === process.pid
|
||||
&& ["not_started", "listening", "closed", "close_failed"].includes(listener.state)
|
||||
&& (listener.state === "not_started" ? !("actualPort" in listener)
|
||||
: Number.isInteger(listener.actualPort) && listener.actualPort >= 1 && listener.actualPort <= 65535);
|
||||
if (value.schemaVersion !== 1 || value.kind !== "p11-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce
|
||||
|| value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid
|
||||
|| !ISO_UTC.test(value.startedAt ?? "") || !validListener
|
||||
|| JSON.stringify(value.resources) !== JSON.stringify(exactOwnedResources(run))) throw new Error("ownership identity mismatch");
|
||||
return value;
|
||||
}
|
||||
export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const id = basename(resolve(runRoot));
|
||||
const lexical = validateRunRoot(repo, runRoot, id);
|
||||
const rootEntry = await lstat(lexical);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory");
|
||||
const ownershipPath = join(lexical, "ownership.json");
|
||||
const ownershipEntry = await lstat(ownershipPath);
|
||||
if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe");
|
||||
let value;
|
||||
try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); }
|
||||
return strictOwnership(value, {
|
||||
repositoryRoot: repo,
|
||||
root: lexical,
|
||||
runId: id,
|
||||
nonce: expectedNonce,
|
||||
startedAt: value.startedAt,
|
||||
pid: process.pid,
|
||||
}, expectedNonce);
|
||||
}
|
||||
export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) {
|
||||
const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce });
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`);
|
||||
await rename(runRoot, tombstone);
|
||||
await rm(tombstone, { recursive: true, force: false });
|
||||
}
|
||||
async function finalizeOwnedRun({ run, success, keep }) {
|
||||
if (!success || keep) return false;
|
||||
await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
return true;
|
||||
}
|
||||
|
||||
function sanitizeForEvidence(value, forbiddenValues = []) {
|
||||
const forbidden = forbiddenValues.filter((item) => typeof item === "string" && item.length > 0);
|
||||
const redactString = (input) => forbidden.reduce((text, secret) => text.split(secret).join("[REDACTED]"), input);
|
||||
if (typeof value === "string") return redactString(value);
|
||||
if (Array.isArray(value)) return value.map((item) => sanitizeForEvidence(item, forbiddenValues));
|
||||
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, sanitizeForEvidence(item, forbiddenValues)]));
|
||||
return value;
|
||||
}
|
||||
async function fileArtifact(root, relativePath) {
|
||||
const bytes = await readFile(join(root, relativePath));
|
||||
return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) };
|
||||
}
|
||||
async function evidence(run, relativePath, value, forbiddenValues = []) {
|
||||
await atomicWrite(join(run.root, relativePath), `${JSON.stringify(sanitizeForEvidence(value, forbiddenValues), null, 2)}\n`);
|
||||
return await fileArtifact(run.root, relativePath);
|
||||
}
|
||||
async function writeJson(path, value) {
|
||||
await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
async function walkFiles(root) {
|
||||
const files = [];
|
||||
async function visit(dir) {
|
||||
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) await visit(path);
|
||||
else if (entry.isFile()) files.push({ path, rel: relative(root, path).split(sep).join("/") });
|
||||
}
|
||||
}
|
||||
if (existsSync(root)) await visit(root);
|
||||
return files.sort((a, b) => a.rel.localeCompare(b.rel));
|
||||
}
|
||||
async function snapshotDigest(root) {
|
||||
const result = {};
|
||||
for (const file of await walkFiles(root)) result[file.rel] = sha256(await readFile(file.path));
|
||||
return result;
|
||||
}
|
||||
function assertByteIdentical(left, right, label) {
|
||||
if (JSON.stringify(left) !== JSON.stringify(right)) throw new Error(`${label} changed unexpectedly`);
|
||||
}
|
||||
async function writeReportFiles({ run, report }) {
|
||||
validateReport(report);
|
||||
await writeJson(join(run.root, "report.json"), report);
|
||||
const lines = [
|
||||
`# P1.1 acceptance report`,
|
||||
"",
|
||||
`Run ID: ${report.runId}`,
|
||||
`Overall: ${report.overall}`,
|
||||
"",
|
||||
...report.checks.map((check) => `- ${check.id}: ${check.status}`),
|
||||
"",
|
||||
`report.json sha256: ${sha256(await readFile(join(run.root, "report.json")))}`,
|
||||
`P1.1 automated integration: ${report.overall}`,
|
||||
"P1.1 manual acceptance: PENDING",
|
||||
];
|
||||
await atomicWrite(join(run.root, "report.md"), `${lines.join("\n")}\n`);
|
||||
}
|
||||
export function validateReport(report) {
|
||||
if (!report || typeof report !== "object" || Array.isArray(report)) throw new Error("report is malformed");
|
||||
if (report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "")
|
||||
|| !ISO_UTC.test(report.finishedAt ?? "") || report.command !== "p11-acceptance integration --keep") throw new Error("report identity is invalid");
|
||||
if (report.overall !== deriveOverall(report.checks ?? [])) throw new Error("report overall is not derived");
|
||||
if (!Array.isArray(report.checks) || report.checks.length !== CHECK_IDS.length) throw new Error("report checks are incomplete");
|
||||
const ids = report.checks.map((check) => check.id);
|
||||
if (JSON.stringify(ids) !== JSON.stringify(CHECK_IDS)) throw new Error("report checks are not exact");
|
||||
const artifactPaths = new Set();
|
||||
for (const check of report.checks) {
|
||||
if (!["PASS", "FAIL"].includes(check.status) || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "")) {
|
||||
throw new Error("report check metadata is invalid");
|
||||
}
|
||||
if (!Array.isArray(check.commands) || check.commands.some((command) => typeof command !== "string" || !/^[A-Za-z0-9._+-]+$/.test(command))) {
|
||||
throw new Error("report command is invalid");
|
||||
}
|
||||
if (!Array.isArray(check.artifacts)) throw new Error("report artifacts are invalid");
|
||||
for (const artifact of check.artifacts) {
|
||||
if (typeof artifact.path !== "string" || artifact.path.startsWith("/") || artifact.path.includes("..") || !/^[A-Za-z0-9._/-]+$/.test(artifact.path)) {
|
||||
throw new Error("report artifact path is invalid");
|
||||
}
|
||||
if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report artifact hash is invalid");
|
||||
if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated");
|
||||
artifactPaths.add(artifact.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function execCommand(executable, argv, { cwd, env, timeoutMs = 30_000, stdin } = {}) {
|
||||
if (!Array.isArray(argv) || argv.some((value) => typeof value !== "string")) throw new Error("command argv must be a string array");
|
||||
const result = await execFileAsync(executable, argv, {
|
||||
cwd,
|
||||
env,
|
||||
timeout: timeoutMs,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
encoding: "utf8",
|
||||
...(stdin === undefined ? {} : { input: stdin }),
|
||||
});
|
||||
return { code: 0, stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
|
||||
}
|
||||
async function git(ctx, argv, options = {}) {
|
||||
return await execCommand(ctx.executables.gitPath, argv, { ...options, env: ctx.env });
|
||||
}
|
||||
async function tht(ctx, argv, options = {}) {
|
||||
try {
|
||||
return await execCommand(ctx.executables.thtPath, argv, { ...options, env: ctx.env });
|
||||
} catch (error) {
|
||||
if (typeof error?.code === "number") return { code: error.code, stdout: error.stdout ?? "", stderr: error.stderr ?? "" };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
|
||||
function baseWorkspace(id, evidenceSource) {
|
||||
return {
|
||||
workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
|
||||
dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
|
||||
};
|
||||
}
|
||||
function descriptors() {
|
||||
return [
|
||||
baseWorkspace("p11-filesystem", { type: "filesystem", uri: "p11-filesystem/evidence", patterns: ["**/*.md"], max_bytes: 10485760 }),
|
||||
baseWorkspace("p11-http", { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", connect_timeout_ms: 1250, read_timeout_ms: 30001, max_bytes: 12345, max_redirects: 2, allow_private_hosts: false, max_cache_bytes: 67890 }),
|
||||
baseWorkspace("p11-s3", { type: "s3", uri: "s3://p11-evidence/published/", endpoint_url: "https://s3.example.test/", region: "eu-west-1", credentials: "static_files", trusted_endpoint: true, allow_private_endpoint: false, allow_insecure_endpoint: false, max_bytes: 12345, max_objects: 33, max_pages: 4, page_size: 5 }),
|
||||
];
|
||||
}
|
||||
async function createTopology(run) {
|
||||
for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 });
|
||||
}
|
||||
async function setupSecrets(ctx) {
|
||||
const secretDir = join(ctx.run.root, "fixture-secrets");
|
||||
const values = {
|
||||
dwh: `DWH-${randomBytes(12).toString("hex")}`,
|
||||
signed: `SIGNED-${randomBytes(12).toString("hex")}`,
|
||||
access: `ACCESS-${randomBytes(12).toString("hex")}`,
|
||||
secret: `SECRET-${randomBytes(12).toString("hex")}`,
|
||||
session: `SESSION-${randomBytes(12).toString("hex")}`,
|
||||
rejected: `REJECTED-${randomBytes(12).toString("hex")}`,
|
||||
};
|
||||
ctx.forbiddenValues = Object.values(values);
|
||||
ctx.secretValues = values;
|
||||
const paths = {
|
||||
dwh: join(secretDir, "dwh-password"),
|
||||
signed: join(secretDir, "evidence-signed-urls.json"),
|
||||
access: join(secretDir, "evidence-access"),
|
||||
secret: join(secretDir, "evidence-secret"),
|
||||
session: join(secretDir, "evidence-session"),
|
||||
};
|
||||
await atomicWrite(paths.dwh, scalarSecretBytes(values.dwh));
|
||||
await atomicWrite(paths.signed, JSON.stringify([`https://evidence.example.test/guide.md?token=${values.signed}`]));
|
||||
await atomicWrite(paths.access, scalarSecretBytes(values.access));
|
||||
await atomicWrite(paths.secret, scalarSecretBytes(values.secret));
|
||||
await atomicWrite(paths.session, scalarSecretBytes(values.session));
|
||||
const env = {};
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const prefix = `THT_WS_${namespace(workspace.workspace.id)}`;
|
||||
Object.assign(env, {
|
||||
[`${prefix}_DWH_TRANSPORT`]: "postgres_direct",
|
||||
[`${prefix}_DWH_HOST`]: "dwh.invalid",
|
||||
[`${prefix}_DWH_PORT`]: "5432",
|
||||
[`${prefix}_DWH_USER`]: "reader",
|
||||
[`${prefix}_DWH_PASSWORD_FILE`]: paths.dwh,
|
||||
});
|
||||
}
|
||||
Object.assign(env, {
|
||||
THT_WS_P11_HTTP_EVIDENCE_SIGNED_URLS_FILE: paths.signed,
|
||||
THT_WS_P11_S3_EVIDENCE_ACCESS_KEY_FILE: paths.access,
|
||||
THT_WS_P11_S3_EVIDENCE_SECRET_KEY_FILE: paths.secret,
|
||||
THT_WS_P11_S3_EVIDENCE_SESSION_TOKEN_FILE: paths.session,
|
||||
});
|
||||
Object.assign(ctx.env, env);
|
||||
await atomicWrite(join(ctx.run.root, "installation", "bindings.env"), `${Object.entries(env).map(([key, value]) => `${key}=${value}`).join("\n")}\n`);
|
||||
await atomicWrite(join(ctx.run.root, "installation", "runtime", "base.yaml"), "{}\n");
|
||||
}
|
||||
function catalog(entries = ctxDescriptors) {
|
||||
return { schema_version: 1, workspaces: entries.map(({ workspace }) => ({ id: workspace.id, name: workspace.name, description: workspace.description })) };
|
||||
}
|
||||
const ctxDescriptors = descriptors();
|
||||
async function initializeGit(ctx) {
|
||||
const author = join(ctx.run.root, "author");
|
||||
await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], { cwd: ctx.run.root });
|
||||
await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], { cwd: ctx.run.root });
|
||||
await git(ctx, ["config", "user.name", "P1 Fixture Curator"], { cwd: author });
|
||||
await git(ctx, ["config", "user.email", "p1-curator@example.invalid"], { cwd: author });
|
||||
const catalogBytes = `${JSON.stringify({
|
||||
schema_version: 1,
|
||||
workspaces: [
|
||||
...catalog(ctx.descriptors).workspaces,
|
||||
{ id: "p11-pending", name: "P1.1 pending", description: "Catalog-only slot awaiting bootstrap" },
|
||||
],
|
||||
}, null, 2)}\n`;
|
||||
await atomicWrite(join(author, "thoth-workspaces.yaml"), catalogBytes, 0o644);
|
||||
const evidenceRoot = join(author, "p11-filesystem", "evidence");
|
||||
await mkdir(join(evidenceRoot, "domain"), { recursive: true });
|
||||
await atomicWrite(join(evidenceRoot, "guide.md"), "# P1.1 curated Evidence\n", 0o644);
|
||||
await atomicWrite(join(evidenceRoot, "domain", "table.md"), "# Curated table\n", 0o644);
|
||||
await git(ctx, ["add", "thoth-workspaces.yaml"], { cwd: author });
|
||||
await git(ctx, ["add", "p11-filesystem/evidence/guide.md"], { cwd: author });
|
||||
await git(ctx, ["add", "-A", "p11-filesystem/evidence"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Bootstrap curated P1 content"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "main"], { cwd: author });
|
||||
ctx.bootstrapCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim();
|
||||
ctx.catalogBlobBefore = (await git(ctx, ["rev-parse", `HEAD:thoth-workspaces.yaml`], { cwd: author })).stdout.trim();
|
||||
ctx.evidenceTreeBefore = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/evidence`], { cwd: author })).stdout.trim();
|
||||
}
|
||||
async function loadProductionBackend() {
|
||||
const [{ loadConfig }, { buildApp }, { WorkspaceRegistry }, { ThtRunner }] = await Promise.all([
|
||||
import("../dist/config.js"),
|
||||
import("../dist/app.js"),
|
||||
import("../dist/workspaces/registry.js"),
|
||||
import("../dist/tht/tht-runner.js"),
|
||||
]);
|
||||
return { loadConfig, buildApp, WorkspaceRegistry, ThtRunner };
|
||||
}
|
||||
async function startBackend(ctx) {
|
||||
const { loadConfig, buildApp, WorkspaceRegistry, ThtRunner } = await loadProductionBackend();
|
||||
const config = loadConfig(ctx.env);
|
||||
const registry = new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const thtRunner = new ThtRunner({
|
||||
thtBin: config.thtBin,
|
||||
harnessDir: config.harnessDir,
|
||||
configPath: join(ctx.run.root, "installation", "runtime", "base.yaml"),
|
||||
dataRoot: config.dataRoot,
|
||||
runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"),
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
secretsFile: config.secretsFile,
|
||||
secretFiles: config.secretFiles,
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
});
|
||||
const app = buildApp(config, { thtRunner, workspaceRegistry: registry });
|
||||
const address = await app.listen({ host: "127.0.0.1", port: 0 });
|
||||
const baseUrl = `http://127.0.0.1:${new URL(address).port}`;
|
||||
ctx.registry = registry;
|
||||
ctx.thtRunner = thtRunner;
|
||||
ctx.app = app;
|
||||
ctx.baseUrl = baseUrl;
|
||||
await writeOwnership(ctx.run, {
|
||||
name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0,
|
||||
actualPort: Number(new URL(address).port), pid: process.pid, state: "listening",
|
||||
});
|
||||
}
|
||||
async function stopBackend(ctx) {
|
||||
if (ctx.app) {
|
||||
await ctx.app.close().catch(() => {});
|
||||
await writeOwnership(ctx.run, {
|
||||
name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0,
|
||||
actualPort: Number(new URL(ctx.baseUrl).port), pid: process.pid, state: "closed",
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
async function request(ctx, id, method, path, body, binary = false, safeInput) {
|
||||
const requestSummary = safeInput === undefined
|
||||
? { method, path, ...(body === undefined ? {} : { body: sanitizeForEvidence(body, ctx.forbiddenValues) }) }
|
||||
: { method, path, input: safeInput };
|
||||
await evidence(ctx.run, `requests/${id}.json`, requestSummary, ctx.forbiddenValues);
|
||||
const response = await fetch(`${ctx.baseUrl}${path}`, {
|
||||
method,
|
||||
headers: body === undefined ? {} : { "content-type": "application/json" },
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
if (binary) {
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
await atomicWrite(join(ctx.run.root, `exports/raw/${id}.zip`), bytes);
|
||||
await evidence(ctx.run, `responses/${id}.json`, { status: response.status, bytes: bytes.length, contentType: response.headers.get("content-type") });
|
||||
return { status: response.status, bytes };
|
||||
}
|
||||
const text = await response.text();
|
||||
let parsed;
|
||||
try { parsed = text ? JSON.parse(text) : null; } catch { parsed = { invalidJson: true, raw: text }; }
|
||||
await evidence(ctx.run, `responses/${id}.json`, { status: response.status, body: sanitizeForEvidence(parsed, ctx.forbiddenValues) }, ctx.forbiddenValues);
|
||||
return { status: response.status, body: parsed };
|
||||
}
|
||||
async function extractZip(ctx, id, bytes) {
|
||||
const yauzl = (await import("yauzl")).default;
|
||||
const output = join(ctx.run.root, "exports", "extracted", id);
|
||||
await mkdir(output, { recursive: true });
|
||||
const files = await new Promise((resolvePromise, reject) => {
|
||||
yauzl.fromBuffer(bytes, { lazyEntries: true, strictFileNames: true, validateEntrySizes: true }, (error, zip) => {
|
||||
if (error || !zip) return reject(error ?? new Error("zip open failed"));
|
||||
const collected = new Map();
|
||||
zip.on("error", reject);
|
||||
zip.on("entry", (entry) => {
|
||||
if (!ZIP_FILES.includes(entry.fileName) || entry.fileName.includes("..") || entry.fileName.startsWith("/") || entry.fileName.endsWith("/")) return reject(new Error("unsafe export entry"));
|
||||
zip.openReadStream(entry, (streamError, stream) => {
|
||||
if (streamError || !stream) return reject(streamError ?? new Error("zip stream failed"));
|
||||
const chunks = [];
|
||||
stream.on("data", (chunk) => chunks.push(chunk));
|
||||
stream.on("error", reject);
|
||||
stream.on("end", async () => {
|
||||
const buffer = Buffer.concat(chunks);
|
||||
collected.set(entry.fileName, buffer);
|
||||
await atomicWrite(join(output, entry.fileName), buffer);
|
||||
zip.readEntry();
|
||||
});
|
||||
});
|
||||
});
|
||||
zip.on("end", () => resolvePromise(collected));
|
||||
zip.readEntry();
|
||||
});
|
||||
});
|
||||
assert(files.size === ZIP_FILES.length, "export bundle entry mismatch");
|
||||
return JSON.parse(files.get("manifest.json").toString("utf8"));
|
||||
}
|
||||
function checkResult(id, startedAt, status, artifacts = [], commands = [], error) {
|
||||
return { id, status, startedAt, finishedAt: nowIso(), artifacts, commands, ...(error ? { error } : {}) };
|
||||
}
|
||||
async function executeChecks({ checks }) {
|
||||
const results = [];
|
||||
let stopped = false;
|
||||
for (const scenario of checks) {
|
||||
const startedAt = nowIso();
|
||||
if (stopped) {
|
||||
results.push(checkResult(scenario.id, startedAt, "FAIL", [], [], "Not executed after earlier failure."));
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const output = await scenario.run();
|
||||
results.push(checkResult(scenario.id, startedAt, "PASS", output.artifacts ?? [], output.commands ?? []));
|
||||
} catch (error) {
|
||||
const partial = error?.acceptancePartial ?? {};
|
||||
results.push(checkResult(scenario.id, startedAt, "FAIL", partial.artifacts ?? [], partial.commands ?? [], "Acceptance scenario failed safely."));
|
||||
stopped = true;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
async function registryState(ctx) {
|
||||
const statePath = join(ctx.run.root, "installation", "registry", "state", "active.json");
|
||||
const active = JSON.parse(await readFile(statePath, "utf8"));
|
||||
return {
|
||||
head: active.head,
|
||||
revisions: active.revisions.map((revision) => ({ id: revision.id, commit: revision.commit, blob: revision.blob })),
|
||||
catalog: active.catalog ?? null,
|
||||
};
|
||||
}
|
||||
function safeErrorEnvelope(response, code, status) {
|
||||
assert(response.status === status, `expected ${status}`);
|
||||
assert(response.body?.code === code, `expected error code ${code}`);
|
||||
assert(Object.keys(response.body).sort().join(",") === "code,message", "error envelope is not exact");
|
||||
}
|
||||
async function productionChecks(ctx) {
|
||||
const check = async (id, value, commands = []) => ({ commands, artifacts: [await evidence(ctx.run, `logs/${id}.json`, value, ctx.forbiddenValues)] });
|
||||
return [
|
||||
{ id: "preflight", run: async () => check("preflight", { node: process.version, repositoryHead: ctx.provenance.head, repositoryTree: ctx.provenance.tree, clean: ctx.provenance.clean, thtExecutable: true }) },
|
||||
{ id: "clean_state", run: async () => check("clean_state", { runId: ctx.run.runId, reused: false }) },
|
||||
{ id: "ownership", run: async () => { await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); return await check("ownership", { valid: true }); } },
|
||||
{ id: "catalog_bootstrap", run: async () => {
|
||||
await initializeGit(ctx);
|
||||
for (const workspace of ctx.descriptors) await atomicWrite(join(ctx.run.root, "fixtures", "descriptors", `${workspace.workspace.id}.json`), `${JSON.stringify(workspace, null, 2)}\n`);
|
||||
return {
|
||||
commands: ["git"],
|
||||
artifacts: [
|
||||
await evidence(ctx.run, "logs/catalog-bootstrap.json", { bootstrapCommit: ctx.bootstrapCommit, catalogOnly: true }),
|
||||
await fileArtifact(ctx.run.root, "author/thoth-workspaces.yaml"),
|
||||
await fileArtifact(ctx.run.root, "author/p11-filesystem/evidence/guide.md"),
|
||||
],
|
||||
};
|
||||
} },
|
||||
{ id: "catalog_only_listing", run: async () => {
|
||||
await startBackend(ctx);
|
||||
const status = await request(ctx, "registry-status", "GET", "/workspace-registry/status");
|
||||
assert(status.status === 200 && status.body.head === ctx.bootstrapCommit, "status head mismatch");
|
||||
const listed = await request(ctx, "workspace-list-initial", "GET", "/workspaces");
|
||||
assert(listed.status === 200 && listed.body.length === 4, "catalog listing failed");
|
||||
assert(listed.body.every((entry) => entry.configurationState === "configuration_required"), "catalog entries were not configuration_required");
|
||||
ctx.baseCommit = status.body.head;
|
||||
return await check("catalog_only_listing", { head: status.body.head, ids: listed.body.map((entry) => entry.id), allConfigurationRequired: true });
|
||||
} },
|
||||
{ id: "bootstrap_create_once", run: async () => {
|
||||
let base = ctx.baseCommit;
|
||||
ctx.bootstrapResponses = {};
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const validated = await request(ctx, `validate-${workspace.workspace.id}`, "POST", "/workspaces/validate", { workspace });
|
||||
assert(validated.status === 200, `validate failed ${workspace.workspace.id}`);
|
||||
const published = await request(ctx, `publish-${workspace.workspace.id}`, "POST", "/workspaces/publish", { action: "create", workspace, baseCommit: base });
|
||||
assert(published.status === 200 && HEX40.test(published.body.revision.commit), `publish failed ${workspace.workspace.id}`);
|
||||
ctx.bootstrapResponses[workspace.workspace.id] = published.body;
|
||||
base = published.body.revision.commit;
|
||||
}
|
||||
ctx.publishHead = base;
|
||||
const listed = await request(ctx, "workspace-list-ready", "GET", "/workspaces");
|
||||
assert(listed.body.filter((entry) => entry.configurationState === "ready").length === 3, "bootstrap did not activate all published entries");
|
||||
assert(listed.body.find((entry) => entry.id === "p11-pending")?.configurationState === "configuration_required", "pending slot was not left unconfigured");
|
||||
return await check("bootstrap_create_once", { head: base, readyIds: listed.body.filter((entry) => entry.configurationState === "ready").map((entry) => entry.id) });
|
||||
} },
|
||||
{ id: "api_curator_boundary", run: async () => {
|
||||
const author = join(ctx.run.root, "author");
|
||||
const catalogAfter = (await git(ctx, ["rev-parse", `HEAD:thoth-workspaces.yaml`], { cwd: author })).stdout.trim();
|
||||
const evidenceAfter = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/evidence`], { cwd: author })).stdout.trim();
|
||||
assert(catalogAfter === ctx.catalogBlobBefore, "catalog blob changed during bootstrap");
|
||||
assert(evidenceAfter === ctx.evidenceTreeBefore, "evidence tree changed during bootstrap");
|
||||
ctx.apiBoundaryState = await registryState(ctx);
|
||||
return await check("api_curator_boundary", { catalogUnchanged: true, evidenceUnchanged: true, state: ctx.apiBoundaryState }, ["git"]);
|
||||
} },
|
||||
{ id: "curator_descriptor_update", run: async () => {
|
||||
const author = join(ctx.run.root, "author");
|
||||
await git(ctx, ["fetch", "origin", "main"], { cwd: author });
|
||||
await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author });
|
||||
const workspace = structuredClone(ctx.descriptors[0]);
|
||||
workspace.workspace.name = "P1.1 Curated Filesystem";
|
||||
workspace.workspace.description = "Curator updated descriptor and catalog metadata";
|
||||
ctx.curatedWorkspace = workspace;
|
||||
const updatedCatalog = catalog([workspace, ctx.descriptors[1], ctx.descriptors[2]]);
|
||||
await atomicWrite(join(author, "thoth-workspaces.yaml"), `${JSON.stringify(updatedCatalog, null, 2)}\n`, 0o644);
|
||||
await atomicWrite(join(author, "p11-filesystem", "workspace.yaml"), `${(await import("yaml")).stringify(workspace)}`, 0o644);
|
||||
await git(ctx, ["add", "thoth-workspaces.yaml"], { cwd: author });
|
||||
await git(ctx, ["add", "--", "p11-filesystem/workspace.yaml"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Publish workspace p1-filesystem"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "main"], { cwd: author });
|
||||
ctx.curatorCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim();
|
||||
ctx.curatorDescriptorBlob = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/workspace.yaml`], { cwd: author })).stdout.trim();
|
||||
const pulled = await request(ctx, "pull-after-curator-update", "POST", "/workspace-registry/pull");
|
||||
assert(pulled.status === 200 && HEX40.test(pulled.body.head), "pull after curator update failed");
|
||||
ctx.docsFollowupHead = pulled.body.head;
|
||||
const read = await request(ctx, "read-after-curator-update", "GET", "/workspaces/p11-filesystem");
|
||||
assert(read.status === 200 && read.body.workspace.workspace.name === workspace.workspace.name, "curator update did not activate");
|
||||
assert(read.body.revision.blob === ctx.curatorDescriptorBlob, "api rewrote curator descriptor bytes");
|
||||
return await check("curator_descriptor_update", { curatorCommit: ctx.curatorCommit, activeHead: ctx.docsFollowupHead, descriptorBlob: ctx.curatorDescriptorBlob }, ["git"]);
|
||||
} },
|
||||
{ id: "content_only_revision", run: async () => {
|
||||
const author = join(ctx.run.root, "author");
|
||||
await git(ctx, ["fetch", "origin", "main"], { cwd: author });
|
||||
await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author });
|
||||
await atomicWrite(join(author, "p11-filesystem", "evidence", "guide.md"), "# P1.1 curated Evidence v2\n", 0o644);
|
||||
await git(ctx, ["add", "p11-filesystem/evidence/guide.md"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Update curated Evidence only"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "main"], { cwd: author });
|
||||
ctx.contentCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim();
|
||||
const pulled = await request(ctx, "pull-after-content-update", "POST", "/workspace-registry/pull");
|
||||
assert(pulled.status === 200 && pulled.body.head === ctx.contentCommit, "content pull head mismatch");
|
||||
const read = await request(ctx, "read-after-content-update", "GET", "/workspaces/p11-filesystem");
|
||||
assert(read.body.revision.commit === ctx.contentCommit, "content commit did not activate");
|
||||
assert(read.body.revision.blob === ctx.curatorDescriptorBlob, "descriptor blob changed on content-only update");
|
||||
ctx.currentRead = read.body;
|
||||
return await check("content_only_revision", { commit: ctx.contentCommit, descriptorBlobUnchanged: true }, ["git"]);
|
||||
} },
|
||||
{ id: "docs_only_reconciliation", run: async () => {
|
||||
const repo = join(ctx.run.root, "installation", "registry", "repo");
|
||||
const diff = (await git(ctx, ["show", "--name-only", "--format=", ctx.docsFollowupHead], { cwd: repo })).stdout.trim().split(/\n+/).filter(Boolean);
|
||||
assert(diff.length > 0 && diff.every((path) => path.startsWith("workspace-docs/")), "docs follow-up touched non-doc paths");
|
||||
const finalDescriptor = (await git(ctx, ["rev-parse", `${ctx.docsFollowupHead}:p11-filesystem/workspace.yaml`], { cwd: repo })).stdout.trim();
|
||||
assert(finalDescriptor === ctx.curatorDescriptorBlob, "docs follow-up rewrote descriptor");
|
||||
return await check("docs_only_reconciliation", { head: ctx.docsFollowupHead, files: diff, descriptorBlobPreserved: true }, ["git"]);
|
||||
} },
|
||||
{ id: "same_revision_git_objects", run: async () => {
|
||||
const repo = join(ctx.run.root, "installation", "registry", "repo");
|
||||
const revision = ctx.currentRead.revision;
|
||||
const manifestPath = join(dirname(revision.snapshotPath), "snapshot.json");
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
||||
const catalogBlob = (await git(ctx, ["rev-parse", `${revision.commit}:thoth-workspaces.yaml`], { cwd: repo })).stdout.trim();
|
||||
const descriptorBlob = (await git(ctx, ["rev-parse", `${revision.commit}:p11-filesystem/workspace.yaml`], { cwd: repo })).stdout.trim();
|
||||
const evidenceTree = (await git(ctx, ["rev-parse", `${revision.commit}:p11-filesystem/evidence`], { cwd: repo })).stdout.trim();
|
||||
assert(manifest.head === revision.commit, "snapshot manifest head mismatch");
|
||||
assert(descriptorBlob === revision.blob, "descriptor blob mismatch");
|
||||
ctx.snapshotManifest = manifest;
|
||||
return {
|
||||
commands: ["git"],
|
||||
artifacts: [
|
||||
await evidence(ctx.run, "logs/same-revision-git-objects.json", { commit: revision.commit, catalogBlob, descriptorBlob, evidenceTree, snapshotHead: manifest.head }),
|
||||
await fileArtifact(ctx.run.root, relative(ctx.run.root, revision.snapshotPath)),
|
||||
await fileArtifact(ctx.run.root, relative(ctx.run.root, manifestPath)),
|
||||
],
|
||||
};
|
||||
} },
|
||||
{ id: "snapshot_and_export", run: async () => {
|
||||
ctx.exportManifests = {};
|
||||
const artifacts = [];
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const id = workspace.workspace.id;
|
||||
const exported = await request(ctx, `export-${id}`, "GET", `/workspaces/${id}/export`, undefined, true);
|
||||
assert(exported.status === 200, `export failed ${id}`);
|
||||
ctx.exportManifests[id] = await extractZip(ctx, id, exported.bytes);
|
||||
artifacts.push(await fileArtifact(ctx.run.root, `exports/raw/export-${id}.zip`));
|
||||
for (const name of ZIP_FILES) artifacts.push(await fileArtifact(ctx.run.root, `exports/extracted/${id}/${name}`));
|
||||
}
|
||||
return { commands: [], artifacts: [await evidence(ctx.run, "logs/snapshot-and-export.json", { exported: Object.keys(ctx.exportManifests), files: ZIP_FILES }), ...artifacts] };
|
||||
} },
|
||||
{ id: "runtime_render_determinism", run: async () => {
|
||||
const YAML = await import("yaml");
|
||||
ctx.configChecks = [];
|
||||
const artifacts = [];
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const revision = (await request(ctx, `read-render-${workspace.workspace.id}`, "GET", `/workspaces/${workspace.workspace.id}`)).body.revision;
|
||||
const renders = [];
|
||||
for (let n = 1; n <= 2; n += 1) {
|
||||
const lease = ctx.thtRunner.acquireWorkspaceRuntime(revision.snapshotPath);
|
||||
try {
|
||||
const bytes = await readFile(lease.path);
|
||||
renders.push(bytes);
|
||||
await atomicWrite(join(ctx.run.root, "rendered", `${workspace.workspace.id}-${n}.yaml`), bytes);
|
||||
const checked = await tht(ctx, ["config", "check", "-c", lease.path], { cwd: ctx.env.THT_HARNESS_DIR, timeoutMs: 30_000 });
|
||||
ctx.configChecks.push({ id: workspace.workspace.id, observation: n, code: checked.code });
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
artifacts.push(await fileArtifact(ctx.run.root, `rendered/${workspace.workspace.id}-${n}.yaml`));
|
||||
}
|
||||
assert(renders[0].equals(renders[1]), `render was nondeterministic ${workspace.workspace.id}`);
|
||||
const rendered = YAML.parse(renders[0].toString("utf8"));
|
||||
assert(rendered.runtime_identity.workspace_revision === revision.commit, `runtime identity mismatch ${workspace.workspace.id}`);
|
||||
}
|
||||
return { commands: ["tht"], artifacts: [await evidence(ctx.run, "logs/runtime-render-determinism.json", { deterministic: true, checks: ctx.configChecks }), ...artifacts] };
|
||||
} },
|
||||
{ id: "tht_config_check", run: async () => {
|
||||
assert(ctx.configChecks.length === ctx.descriptors.length * 2 && ctx.configChecks.every((item) => item.code === 0), "tht config checks failed");
|
||||
return await check("tht-config-check", ctx.configChecks, ["tht"]);
|
||||
} },
|
||||
{ id: "negative_catalog_layout_cases", run: async () => {
|
||||
const baseline = await registryState(ctx);
|
||||
const author = join(ctx.run.root, "author");
|
||||
const current = (await request(ctx, "current-list-before-negatives", "GET", "/workspaces")).body;
|
||||
const secondCreate = await request(ctx, "second-create", "POST", "/workspaces/publish", { action: "create", workspace: ctx.descriptors[0], baseCommit: baseline.head });
|
||||
safeErrorEnvelope(secondCreate, "workspace_curator_owned", 409);
|
||||
const update = await request(ctx, "legacy-update", "POST", "/workspaces/publish", { action: "update", workspace: ctx.descriptors[0], baseCommit: baseline.head, baseBlob: ctx.curatorDescriptorBlob });
|
||||
safeErrorEnvelope(update, "workspace_curator_owned", 409);
|
||||
const deletion = await request(ctx, "legacy-delete", "POST", "/workspaces/publish", { action: "delete", id: "p11-filesystem", baseCommit: baseline.head, baseBlob: ctx.curatorDescriptorBlob });
|
||||
safeErrorEnvelope(deletion, "workspace_curator_owned", 409);
|
||||
const unknown = structuredClone(ctx.descriptors[0]);
|
||||
unknown.workspace.id = "p11-unknown";
|
||||
const unknownPublish = await request(ctx, "unknown-catalog-id", "POST", "/workspaces/publish", { action: "create", workspace: unknown, baseCommit: baseline.head });
|
||||
safeErrorEnvelope(unknownPublish, "workspace_invalid", 400);
|
||||
const mismatch = structuredClone(ctx.descriptors[0]);
|
||||
mismatch.workspace.id = "p11-pending";
|
||||
mismatch.workspace.name = "Mismatched pending name";
|
||||
mismatch.semantic_index.vector_store.collection = "p11-pending";
|
||||
const mismatchPublish = await request(ctx, "catalog-metadata-mismatch", "POST", "/workspaces/publish", { action: "create", workspace: mismatch, baseCommit: baseline.head });
|
||||
safeErrorEnvelope(mismatchPublish, "workspace_invalid", 400);
|
||||
const after = await registryState(ctx);
|
||||
assertByteIdentical(after, baseline, "registry state after curator-owned refusals");
|
||||
assert(JSON.stringify((await request(ctx, "current-list-after-negatives", "GET", "/workspaces")).body) === JSON.stringify(current), "workspace listing mutated after negative cases");
|
||||
await git(ctx, ["fetch", "origin", "main"], { cwd: author });
|
||||
await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author });
|
||||
await mkdir(join(author, "workspaces"), { recursive: true });
|
||||
await atomicWrite(join(author, "workspaces", "legacy.yaml"), "workspace: bad\n", 0o644);
|
||||
await git(ctx, ["add", "--", "workspaces/legacy.yaml"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Invalid contextual Evidence state"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "HEAD:main"], { cwd: author });
|
||||
const rejectedPull = await request(ctx, "invalid-layout-pull", "POST", "/workspace-registry/pull");
|
||||
safeErrorEnvelope(rejectedPull, "workspace_invalid", 400);
|
||||
const afterInvalidPull = await registryState(ctx);
|
||||
assertByteIdentical(afterInvalidPull, baseline, "registry state after invalid pull");
|
||||
return await check("negative_catalog_layout_cases", { secondCreate: true, update: true, delete: true, unknownCatalogId: true, metadataMismatch: true, oldLayoutRejected: true }, ["git"]);
|
||||
} },
|
||||
{ id: "negative_schema_context_cases", run: async () => {
|
||||
const base = structuredClone(ctx.descriptors[0]);
|
||||
const cases = [
|
||||
["invalid-uri", (workspace) => { workspace.evidence.source.uri = "/etc/passwd"; }, "evidence.source.uri"],
|
||||
["invalid-secret-field", (workspace) => { workspace.evidence.source.password = ctx.secretValues.rejected; }, "evidence.source.password"],
|
||||
["missing-evidence-tree", (workspace) => { workspace.workspace.id = "p11-pending"; workspace.workspace.name = "P1.1 pending"; workspace.workspace.description = "Catalog-only slot awaiting bootstrap"; workspace.semantic_index.vector_store.collection = "p11-pending"; workspace.evidence.source.uri = "p11-pending/evidence"; }, "evidence.source.uri"],
|
||||
];
|
||||
const outcomes = [];
|
||||
for (const [id, mutate, field] of cases) {
|
||||
const workspace = structuredClone(base);
|
||||
mutate(workspace);
|
||||
const endpoint = id === "missing-evidence-tree" ? "/workspaces/publish" : "/workspaces/validate";
|
||||
const payload = id === "missing-evidence-tree" ? { action: "create", workspace, baseCommit: ctx.publishHead } : { workspace };
|
||||
const response = await request(ctx, `negative-schema-${id}`, "POST", endpoint, payload, false, { case: id, expectedInputField: field });
|
||||
safeErrorEnvelope(response, "workspace_invalid", 400);
|
||||
outcomes.push({ case: id, status: response.status, field });
|
||||
}
|
||||
return await check("negative_schema_context_cases", outcomes);
|
||||
} },
|
||||
{ id: "no_p2_scope_artifacts", run: async () => {
|
||||
const forbidden = ["artifacts/evidence", "materialized", "qdrant", "embedding", "ACTIVE", "retention"];
|
||||
const present = forbidden.filter((path) => existsSync(join(ctx.run.root, path)));
|
||||
assert(present.length === 0, "p2 scope artifacts present");
|
||||
return await check("no_p2_scope_artifacts", { absent: forbidden });
|
||||
} },
|
||||
{ id: "secret_scan", run: async () => {
|
||||
const findings = await scanSecrets({ runRoot: ctx.run.root, forbiddenValues: ctx.forbiddenValues, expectedGitRepositories: ["remote.git", "author"] });
|
||||
assert(findings.length === 0, "secret scan found leaked secret material");
|
||||
return await check("secret_scan", { findings: 0 });
|
||||
} },
|
||||
{ id: "cleanup_confinement", run: async () => {
|
||||
const parent = canonicalIntegrationBase(ctx.repositoryRoot);
|
||||
const siblings = (await readdir(parent)).filter((name) => name !== ctx.run.runId);
|
||||
return await check("cleanup_confinement", { listenerState: ctx.run.listeners[0].state, siblingCount: siblings.length });
|
||||
} },
|
||||
];
|
||||
}
|
||||
|
||||
async function setupContext({ repositoryRoot = defaultRepositoryRoot, env = process.env } = {}) {
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const provenance = await collectRepositoryProvenance({ repositoryRoot });
|
||||
const executables = resolveExecutables(repositoryRoot);
|
||||
const harnessDir = realpathSync(join(repositoryRoot, "harness"));
|
||||
const ownedHome = join(run.root, "installation", "runtime", "acceptance-home");
|
||||
const ownedTmp = join(run.root, "installation", "runtime", "tmp");
|
||||
await mkdir(ownedHome, { recursive: true, mode: 0o700 });
|
||||
await mkdir(ownedTmp, { recursive: true, mode: 0o700 });
|
||||
const executablePath = [...new Set([dirname(executables.gitPath), dirname(executables.pythonPath), dirname(executables.thtPath)])].join(":");
|
||||
const fixtureEnv = {
|
||||
PATH: executablePath,
|
||||
HOME: ownedHome,
|
||||
TMPDIR: ownedTmp,
|
||||
HOST: "127.0.0.1",
|
||||
PORT: "0",
|
||||
AUTH_MODE: "none",
|
||||
THT_BIN: executables.thtPath,
|
||||
THT_HARNESS_DIR: harnessDir,
|
||||
THT_DATA_ROOT: join(run.root, "installation", "data"),
|
||||
SETTINGS_FILE: join(run.root, "installation", "data", "settings.json"),
|
||||
MAINTENANCE_STATE_FILE: join(run.root, "installation", "data", "maintenance.json"),
|
||||
THT_WORKSPACE_REGISTRY_ROOT: join(run.root, "installation", "registry"),
|
||||
THT_WORKSPACE_GIT_REMOTE: join(run.root, "remote.git"),
|
||||
THT_WORKSPACE_GIT_BRANCH: "main",
|
||||
THT_WORKSPACE_GIT_AUTHOR_NAME: "P1 API Publisher",
|
||||
THT_WORKSPACE_GIT_AUTHOR_EMAIL: "p1-api@example.invalid",
|
||||
THT_WORKSPACE_INSTALLATION_ID: "p11-acceptance",
|
||||
THT_WORKSPACE_SECRET_ROOTS: join(run.root, "fixture-secrets"),
|
||||
THT_HOME: join(run.root, "installation", "runtime", "tht-home"),
|
||||
PYTHONDONTWRITEBYTECODE: "1",
|
||||
PYTHONNOUSERSITE: "1",
|
||||
};
|
||||
const ctx = {
|
||||
run,
|
||||
repositoryRoot: canonicalRoot(repositoryRoot),
|
||||
provenance,
|
||||
executables,
|
||||
descriptors: descriptors(),
|
||||
env: buildSafeEnvironment({ ambient: env, fixture: fixtureEnv }),
|
||||
forbiddenValues: [],
|
||||
};
|
||||
await createTopology(run);
|
||||
await setupSecrets(ctx);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) {
|
||||
const ctx = await setupContext({ repositoryRoot, env });
|
||||
const priorEnv = {};
|
||||
for (const [key, value] of Object.entries(ctx.env)) {
|
||||
priorEnv[key] = process.env[key];
|
||||
process.env[key] = value;
|
||||
}
|
||||
let success = false;
|
||||
try {
|
||||
const checks = await productionChecks(ctx);
|
||||
const results = await executeChecks({ checks });
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: ctx.run.runId,
|
||||
startedAt: ctx.run.startedAt,
|
||||
finishedAt: nowIso(),
|
||||
command: "p11-acceptance integration --keep",
|
||||
overall: deriveOverall(results),
|
||||
checks: results,
|
||||
};
|
||||
await writeReportFiles({ run: ctx.run, report });
|
||||
success = report.overall === "PASS";
|
||||
if (announce) await announce({ report, runRoot: ctx.run.root });
|
||||
return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) };
|
||||
} finally {
|
||||
await stopBackend(ctx).catch(() => {});
|
||||
for (const [key, value] of Object.entries(ctx.env)) {
|
||||
if (priorEnv[key] === undefined) delete process.env[key];
|
||||
else process.env[key] = priorEnv[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), env = process.env) {
|
||||
if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) {
|
||||
throw new Error("usage: p11-acceptance.mjs integration [--keep]");
|
||||
}
|
||||
const result = await runIntegration({ keep: argv.includes("--keep"), env });
|
||||
return result.exitCode;
|
||||
}
|
||||
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) {
|
||||
try {
|
||||
const code = await main();
|
||||
process.exitCode = code;
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p11-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p11-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p11-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p11 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p11-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p11-${"A".repeat(32)}`), `p11-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p11-${"c".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p11 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p11-${"d".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-11T00:00:00.000Z",
|
||||
finishedAt: "2026-08-11T00:00:01.000Z",
|
||||
commands: ["git"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p11 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p11-${"e".repeat(32)}`,
|
||||
startedAt: "2026-08-11T00:00:00.000Z",
|
||||
finishedAt: "2026-08-11T00:00:10.000Z",
|
||||
command: "p11-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p1-${"e".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p11-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P11_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
@@ -1,404 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { closeSync, constants as fsConstants, fsyncSync, lstatSync, openSync, realpathSync } from "node:fs";
|
||||
import { access, lstat, mkdir, open, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { execFile } from "node:child_process";
|
||||
import http from "node:http";
|
||||
|
||||
import { buildSafeEnvironment } from "./p1-acceptance.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const HOST = "127.0.0.1";
|
||||
const BACKEND_PORT = 8791;
|
||||
const FRONTEND_PORT = 8792;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
const OWNERSHIP_DIGEST = "ownership.sha256";
|
||||
|
||||
function resolveSystemExecutable(name) {
|
||||
for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) {
|
||||
try {
|
||||
const resolved = realpathSync(candidate);
|
||||
if (lstatSync(resolved).isFile()) return resolved;
|
||||
} catch {}
|
||||
}
|
||||
throw new Error(`required executable not found: ${name}`);
|
||||
}
|
||||
function resolveExecutables(repositoryRoot) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const thtPath = join(repo, "harness", ".venv", "bin", "tht");
|
||||
if (!lstatSync(thtPath).isFile()) throw new Error("required executable not found: tht");
|
||||
return { gitPath: resolveSystemExecutable("git"), pythonPath: resolveSystemExecutable("python3"), thtPath: realpathSync(thtPath) };
|
||||
}
|
||||
|
||||
function nowIso() { return new Date().toISOString(); }
|
||||
function fixedManualRoot(repositoryRoot = defaultRepositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p11"); }
|
||||
function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); }
|
||||
function noSymlinkExisting(repo, target) {
|
||||
const rel = relative(repo, target);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("root leaves repository");
|
||||
let cursor = repo;
|
||||
for (const part of rel.split(sep).filter(Boolean)) {
|
||||
cursor = join(cursor, part);
|
||||
if (!lstatSync(cursor, { throwIfNoEntry: false })) break;
|
||||
if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink");
|
||||
}
|
||||
}
|
||||
async function atomicWrite(path, bytes, mode = 0o600) {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`);
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(staging, "wx", mode);
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
await rename(staging, path);
|
||||
const directory = openSync(dirname(path), fsConstants.O_RDONLY);
|
||||
try { fsyncSync(directory); } finally { closeSync(directory); }
|
||||
} catch (error) {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
await rm(staging, { force: true }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function ownershipDigest(bytes) { return createHash("sha256").update(bytes).digest("hex"); }
|
||||
async function writeManualOwnership(root, value) {
|
||||
const body = `${JSON.stringify(value, null, 2)}\n`;
|
||||
await atomicWrite(join(root, "ownership.json"), body);
|
||||
await atomicWrite(join(root, OWNERSHIP_DIGEST), `${ownershipDigest(body)}\n`);
|
||||
}
|
||||
async function git(executable, argv, options = {}) {
|
||||
const result = await execFileAsync(executable, argv, { cwd: options.cwd, env: options.env, timeout: options.timeoutMs ?? 30_000, maxBuffer: 8 * 1024 * 1024, encoding: "utf8" });
|
||||
return { stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
|
||||
}
|
||||
function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
|
||||
function baseWorkspace(id, evidenceSource) {
|
||||
return {
|
||||
workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
|
||||
dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
|
||||
};
|
||||
}
|
||||
function descriptors() {
|
||||
return [
|
||||
baseWorkspace("p11-filesystem", { type: "filesystem", uri: "p11-filesystem/evidence", patterns: ["**/*.md"], max_bytes: 10485760 }),
|
||||
baseWorkspace("p11-http", { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", connect_timeout_ms: 1250, read_timeout_ms: 30001, max_bytes: 12345, max_redirects: 2, allow_private_hosts: false, max_cache_bytes: 67890 }),
|
||||
baseWorkspace("p11-s3", { type: "s3", uri: "s3://p11-evidence/published/", endpoint_url: "https://s3.example.test/", region: "eu-west-1", credentials: "static_files", trusted_endpoint: true, allow_private_endpoint: false, allow_insecure_endpoint: false, max_bytes: 12345, max_objects: 33, max_pages: 4, page_size: 5 }),
|
||||
];
|
||||
}
|
||||
function catalog(entries) {
|
||||
return { schema_version: 1, workspaces: entries.map(({ workspace }) => ({ id: workspace.id, name: workspace.name, description: workspace.description })) };
|
||||
}
|
||||
function quote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; }
|
||||
function requestFixtures(items) {
|
||||
const fixtures = { "status.json": { method: "GET", path: "/workspace-registry/status" }, "pull.json": { method: "POST", path: "/workspace-registry/pull" } };
|
||||
for (const workspace of items) {
|
||||
const id = workspace.workspace.id;
|
||||
fixtures[`validate-${id}.json`] = { workspace };
|
||||
fixtures[`publish-${id}.json`] = { action: "create", workspace };
|
||||
fixtures[`read-${id}.json`] = { method: "GET", path: `/workspaces/${id}` };
|
||||
fixtures[`export-${id}.json`] = { method: "GET", path: `/workspaces/${id}/export` };
|
||||
}
|
||||
fixtures["negative-invalid-uri.json"] = { workspace: { ...items[0], evidence: { ...items[0].evidence, source: { ...items[0].evidence.source, uri: "/etc/passwd" } } } };
|
||||
fixtures["negative-secret-field.json"] = { workspace: { ...items[2], evidence: { ...items[2].evidence, source: { ...items[2].evidence.source, access_key: "CANARY-MUST-BE-REJECTED" } } } };
|
||||
return fixtures;
|
||||
}
|
||||
function curlGet(url, output) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; }
|
||||
function curlPost(url, output, body) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(body)} --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; }
|
||||
function curlPostEmpty(url, output) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; }
|
||||
function publishCurl(root, id, previousResponse) {
|
||||
const descriptor = join(root, "requests", `publish-${id}.json`);
|
||||
const response = join(root, "responses", `publish-${id}.json`);
|
||||
return `#!/usr/bin/env bash\nset -euo pipefail\nbase_commit=$(node -e 'const fs=require("node:fs");const value=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));console.log(value.head ?? value.revision?.commit ?? "");' ${quote(previousResponse)})\nnode -e 'const fs=require("node:fs");const body=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));body.baseCommit=process.argv[2];fs.writeFileSync(process.argv[1],JSON.stringify(body,null,2)+"\\n");' ${quote(descriptor)} "$base_commit"\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(descriptor)} --output ${quote(response)} --write-out 'HTTP %{http_code}\\n' 'http://${HOST}:${BACKEND_PORT}/workspaces/publish'\n`; }
|
||||
function renderCommand(repo, root, observation) {
|
||||
const readResponse = join(root, "responses", "read-p11-filesystem.json");
|
||||
const output = join(root, "rendered", `runtime-${observation}.yaml`);
|
||||
return `#!/usr/bin/env bash\nset -euo pipefail\nread_snapshot=$(node -e 'const fs=require("node:fs");const read=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));const path=read.revision.snapshotPath;const manifest=JSON.parse(fs.readFileSync(require("node:path").join(require("node:path").dirname(path),"snapshot.json"),"utf8"));const name=require("node:path").basename(path);console.log(JSON.stringify({snapshot:path,digest:manifest.files[name]}));' ${quote(readResponse)})\nsnapshot=$(node -e 'const value=JSON.parse(process.argv[1]);console.log(value.snapshot)' "$read_snapshot")\ndigest=$(node -e 'const value=JSON.parse(process.argv[1]);console.log(value.digest)' "$read_snapshot")\nnode ${quote(join(repo, "backend", "scripts", "p11-render-snapshot.mjs"))} --ownership ${quote(join(root, "ownership.json"))} --snapshot "$snapshot" --output ${quote(output)} --snapshot-sha256 "$digest"\n`;
|
||||
}
|
||||
function guide(root) {
|
||||
return `# P1.1 manual acceptance guide
|
||||
|
||||
1. Inspect ${join(root, "ownership.json")}, ${join(root, "author", "thoth-workspaces.yaml")}, nested workspace directories, evidence tree, and fixture secret paths without printing secret bytes.
|
||||
2. Run ./scripts/p11-manual-acceptance.sh serve and confirm only ${HOST}:${BACKEND_PORT} and ${HOST}:${FRONTEND_PORT} are listening for this lab.
|
||||
3. Run commands/http-01-status.sh and inspect responses/status.json plus GET /workspaces for configuration_required slots.
|
||||
4. Run the validate and publish scripts once per slot in numeric order.
|
||||
5. Inspect Git object IDs for thoth-workspaces.yaml, <id>/workspace.yaml, <id>/evidence, and workspace-docs/<id>.
|
||||
6. Retry create/update/delete and verify refusal plus unchanged object IDs.
|
||||
7. In ${join(root, "author")}, edit p11-filesystem/workspace.yaml and thoth-workspaces.yaml together, commit, push, then run commands/http-08-pull.sh and verify the API activated curator bytes without rewriting the descriptor.
|
||||
8. Make an evidence-only commit under p11-filesystem/evidence, push, pull, and inspect the new revision commit with unchanged descriptor blob.
|
||||
9. In the UI at http://${HOST}:${FRONTEND_PORT}, confirm ready workspaces are read-only and bootstrap-only slots are editable before creation.
|
||||
10. Export/import only under bootstrap rules.
|
||||
11. Run commands/render-1.sh and commands/render-2.sh, diff rendered/runtime-1.yaml rendered/runtime-2.yaml, then run tht config check -c on both outputs.
|
||||
12. Run the negative validate scripts and a bounded secret scan outside fixture-secrets.
|
||||
13. Run ./scripts/p11-manual-acceptance.sh stop, verify cleanup of both listeners, write VERDICT.md yourself, and run cleanup only when evidence is no longer needed.
|
||||
`;
|
||||
}
|
||||
function ownershipValue(root, repositoryRoot, nonce, extras = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "p11-manual-acceptance",
|
||||
nonce,
|
||||
repositoryRoot,
|
||||
root,
|
||||
createdAt: nowIso(),
|
||||
status: "PENDING",
|
||||
listeners: {
|
||||
backend: { host: HOST, port: BACKEND_PORT },
|
||||
frontend: { host: HOST, port: FRONTEND_PORT },
|
||||
},
|
||||
resources: [root, join(root, "remote.git"), join(root, "author"), join(root, "fixture-secrets")],
|
||||
...extras,
|
||||
};
|
||||
}
|
||||
export async function readManualOwnership({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
noSymlinkExisting(repo, root);
|
||||
const rootEntry = await lstat(root);
|
||||
const ownershipPath = join(root, "ownership.json");
|
||||
const digestPath = join(root, OWNERSHIP_DIGEST);
|
||||
const ownershipEntry = await lstat(ownershipPath);
|
||||
const digestEntry = await lstat(digestPath);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink() || !digestEntry.isFile() || digestEntry.isSymbolicLink()) throw new Error("manual ownership is unsafe");
|
||||
const ownershipBytes = await readFile(ownershipPath, "utf8");
|
||||
const recordedDigest = (await readFile(digestPath, "utf8")).trim();
|
||||
if (!HEX64.test(recordedDigest) || recordedDigest !== ownershipDigest(ownershipBytes)) throw new Error("manual ownership digest mismatch");
|
||||
const value = JSON.parse(ownershipBytes);
|
||||
if (value?.schemaVersion !== 1 || value.kind !== "p11-manual-acceptance" || !HEX64.test(value.nonce ?? "") || value.repositoryRoot !== repo || value.root !== root) {
|
||||
throw new Error("manual ownership identity mismatch");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
async function ensureRootAbsent(root) {
|
||||
try { await lstat(root); throw new Error("manual acceptance root already exists"); } catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
}
|
||||
async function waitForHttp(url, timeoutMs = 15_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
await new Promise((resolvePromise, reject) => {
|
||||
const request = http.get(url, (response) => { response.resume(); response.statusCode && response.statusCode < 500 ? resolvePromise() : reject(new Error("not ready")); });
|
||||
request.on("error", reject);
|
||||
});
|
||||
return;
|
||||
} catch {
|
||||
await new Promise((resolvePromise) => setTimeout(resolvePromise, 250));
|
||||
}
|
||||
}
|
||||
throw new Error(`timed out waiting for ${url}`);
|
||||
}
|
||||
function live(pid) { try { process.kill(pid, 0); return true; } catch { return false; } }
|
||||
async function writeCommands(repo, root) {
|
||||
const commands = [
|
||||
["http-01-status.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspace-registry/status`, join(root, "responses", "status.json"))],
|
||||
["http-02-validate-p11-filesystem.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-filesystem.json"), join(root, "requests", "validate-p11-filesystem.json"))],
|
||||
["http-03-validate-p11-http.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-http.json"), join(root, "requests", "validate-p11-http.json"))],
|
||||
["http-04-validate-p11-s3.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-s3.json"), join(root, "requests", "validate-p11-s3.json"))],
|
||||
["http-05-publish-p11-filesystem.sh", publishCurl(root, "p11-filesystem", join(root, "responses", "status.json"))],
|
||||
["http-06-publish-p11-http.sh", publishCurl(root, "p11-http", join(root, "responses", "publish-p11-filesystem.json"))],
|
||||
["http-07-publish-p11-s3.sh", publishCurl(root, "p11-s3", join(root, "responses", "publish-p11-http.json"))],
|
||||
["http-08-pull.sh", curlPostEmpty(`http://${HOST}:${BACKEND_PORT}/workspace-registry/pull`, join(root, "responses", "pull.json"))],
|
||||
["http-09-read-p11-filesystem.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspaces/p11-filesystem`, join(root, "responses", "read-p11-filesystem.json"))],
|
||||
["http-10-export-p11-filesystem.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspaces/p11-filesystem/export`, join(root, "exports", "raw", "p11-filesystem.zip"))],
|
||||
["http-11-negative-invalid-uri.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "negative-invalid-uri.json"), join(root, "requests", "negative-invalid-uri.json"))],
|
||||
["http-12-negative-secret-field.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "negative-secret-field.json"), join(root, "requests", "negative-secret-field.json"))],
|
||||
["render-1.sh", renderCommand(repo, root, 1)],
|
||||
["render-2.sh", renderCommand(repo, root, 2)],
|
||||
];
|
||||
for (const [name, body] of commands) {
|
||||
const path = join(root, "commands", name);
|
||||
await atomicWrite(path, body, 0o700);
|
||||
}
|
||||
}
|
||||
export async function prepareManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
noSymlinkExisting(repo, root);
|
||||
await ensureRootAbsent(root);
|
||||
await mkdir(join(repo, ".artifacts", "manual-acceptance"), { recursive: true, mode: 0o700 });
|
||||
await mkdir(root, { mode: 0o700 });
|
||||
const executables = resolveExecutables(repo);
|
||||
const nonce = randomBytes(32).toString("hex");
|
||||
await writeManualOwnership(root, ownershipValue(root, repo, nonce));
|
||||
for (const path of ["fixture-secrets", "requests", "responses", "commands", "rendered", "logs", "exports/raw", "exports/extracted", "installation/registry", "installation/data", "installation/runtime"]) {
|
||||
await mkdir(join(root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 });
|
||||
}
|
||||
const env = buildSafeEnvironment({ ambient: process.env, fixture: { PATH: dirname(executables.gitPath) } });
|
||||
await git(executables.gitPath, ["init", "--bare", "--initial-branch=main", join(root, "remote.git")], { cwd: root, env });
|
||||
await git(executables.gitPath, ["clone", join(root, "remote.git"), join(root, "author")], { cwd: root, env });
|
||||
await git(executables.gitPath, ["config", "user.name", "P1 Fixture Curator"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["config", "user.email", "p1-curator@example.invalid"], { cwd: join(root, "author"), env });
|
||||
const items = descriptors();
|
||||
await atomicWrite(join(root, "author", "thoth-workspaces.yaml"), `${JSON.stringify(catalog(items), null, 2)}\n`, 0o644);
|
||||
await mkdir(join(root, "author", "p11-filesystem", "evidence", "domain"), { recursive: true });
|
||||
await atomicWrite(join(root, "author", "p11-filesystem", "evidence", "guide.md"), "# P1.1 curated Evidence\n", 0o644);
|
||||
await atomicWrite(join(root, "author", "p11-filesystem", "evidence", "domain", "table.md"), "# Curated table\n", 0o644);
|
||||
await git(executables.gitPath, ["add", "thoth-workspaces.yaml"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["add", "-A", "p11-filesystem/evidence"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["commit", "-m", "Bootstrap curated P1 content"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["push", "origin", "main"], { cwd: join(root, "author"), env });
|
||||
const secrets = {
|
||||
dwh: join(root, "fixture-secrets", "dwh-password"),
|
||||
signed: join(root, "fixture-secrets", "evidence-signed-urls.json"),
|
||||
access: join(root, "fixture-secrets", "evidence-access"),
|
||||
secret: join(root, "fixture-secrets", "evidence-secret"),
|
||||
session: join(root, "fixture-secrets", "evidence-session"),
|
||||
};
|
||||
await atomicWrite(secrets.dwh, "manual-dwh-secret", 0o600);
|
||||
await atomicWrite(secrets.signed, JSON.stringify(["https://evidence.example.test/guide.md?token=manual"]), 0o600);
|
||||
await atomicWrite(secrets.access, "manual-access", 0o600);
|
||||
await atomicWrite(secrets.secret, "manual-secret", 0o600);
|
||||
await atomicWrite(secrets.session, "manual-session", 0o600);
|
||||
const bindings = {};
|
||||
for (const workspace of items) {
|
||||
const prefix = `THT_WS_${namespace(workspace.workspace.id)}`;
|
||||
Object.assign(bindings, {
|
||||
[`${prefix}_DWH_TRANSPORT`]: "postgres_direct",
|
||||
[`${prefix}_DWH_HOST`]: "dwh.invalid",
|
||||
[`${prefix}_DWH_PORT`]: "5432",
|
||||
[`${prefix}_DWH_USER`]: "reader",
|
||||
[`${prefix}_DWH_PASSWORD_FILE`]: secrets.dwh,
|
||||
});
|
||||
}
|
||||
Object.assign(bindings, {
|
||||
THT_WS_P11_HTTP_EVIDENCE_SIGNED_URLS_FILE: secrets.signed,
|
||||
THT_WS_P11_S3_EVIDENCE_ACCESS_KEY_FILE: secrets.access,
|
||||
THT_WS_P11_S3_EVIDENCE_SECRET_KEY_FILE: secrets.secret,
|
||||
THT_WS_P11_S3_EVIDENCE_SESSION_TOKEN_FILE: secrets.session,
|
||||
});
|
||||
await atomicWrite(join(root, "installation", "bindings.env"), `${Object.entries(bindings).map(([key, value]) => `${key}=${value}`).join("\n")}\n`);
|
||||
await atomicWrite(join(root, "installation", "runtime", "base.yaml"), "{}\n");
|
||||
for (const [name, value] of Object.entries(requestFixtures(items))) await atomicWrite(join(root, "requests", name), `${JSON.stringify(value, null, 2)}\n`, 0o600);
|
||||
await writeCommands(repo, root);
|
||||
await atomicWrite(join(root, "GUIDE.md"), guide(root), 0o600);
|
||||
await atomicWrite(join(root, "logs", "backend.log"), "", 0o600);
|
||||
const current = await readManualOwnership({ repositoryRoot: repo });
|
||||
current.status = "PENDING";
|
||||
current.requestFixtures = Object.keys(requestFixtures(items));
|
||||
current.commandScripts = (await readdir(join(root, "commands"))).sort();
|
||||
await writeManualOwnership(root, current);
|
||||
return root;
|
||||
}
|
||||
export async function serveManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repo });
|
||||
if (owned.status === "RUNNING") throw new Error("manual acceptance is already serving");
|
||||
await access(join(repo, "backend", "dist", "server.js"));
|
||||
await access(join(repo, "frontend", "dist", "index.html"));
|
||||
const executables = resolveExecutables(repo);
|
||||
const logHandle = await open(join(root, "logs", "backend.log"), fsConstants.O_WRONLY | fsConstants.O_APPEND);
|
||||
const homeDir = join(root, "installation", "runtime", "home");
|
||||
const tmpDir = join(root, "installation", "runtime", "tmp");
|
||||
await mkdir(homeDir, { recursive: true, mode: 0o700 });
|
||||
await mkdir(tmpDir, { recursive: true, mode: 0o700 });
|
||||
const fixtureEnv = {
|
||||
PATH: `${dirname(executables.gitPath)}:${dirname(executables.pythonPath)}:${dirname(executables.thtPath)}:/usr/bin:/bin`,
|
||||
HOME: homeDir,
|
||||
TMPDIR: tmpDir,
|
||||
HOST,
|
||||
PORT: String(BACKEND_PORT),
|
||||
AUTH_MODE: "none",
|
||||
THT_BIN: executables.thtPath,
|
||||
THT_HARNESS_DIR: join(repo, "harness"),
|
||||
THT_DATA_ROOT: join(root, "installation", "data"),
|
||||
SETTINGS_FILE: join(root, "installation", "data", "settings.json"),
|
||||
MAINTENANCE_STATE_FILE: join(root, "installation", "data", "maintenance.json"),
|
||||
THT_WORKSPACE_REGISTRY_ROOT: join(root, "installation", "registry"),
|
||||
THT_WORKSPACE_GIT_REMOTE: join(root, "remote.git"),
|
||||
THT_WORKSPACE_GIT_BRANCH: "main",
|
||||
THT_WORKSPACE_GIT_AUTHOR_NAME: "P1 API Publisher",
|
||||
THT_WORKSPACE_GIT_AUTHOR_EMAIL: "p1-api@example.invalid",
|
||||
THT_WORKSPACE_INSTALLATION_ID: "p11-manual-acceptance",
|
||||
THT_WORKSPACE_SECRET_ROOTS: join(root, "fixture-secrets"),
|
||||
THT_HOME: join(root, "installation", "runtime", "tht-home"),
|
||||
PYTHONDONTWRITEBYTECODE: "1",
|
||||
PYTHONNOUSERSITE: "1",
|
||||
};
|
||||
const bindingEnv = Object.fromEntries((await readFile(join(root, "installation", "bindings.env"), "utf8")).trim().split(/\n+/).map((line) => line.split(/=(.+)/)));
|
||||
const env = buildSafeEnvironment({ ambient: process.env, fixture: { ...fixtureEnv, ...bindingEnv } });
|
||||
const backend = spawn(process.execPath, [join(repo, "backend", "dist", "server.js")], { cwd: repo, env, stdio: ["ignore", logHandle.fd, logHandle.fd], detached: true });
|
||||
const frontend = spawn(executables.pythonPath, ["-m", "http.server", String(FRONTEND_PORT), "--bind", HOST, "--directory", join(repo, "frontend", "dist")], { cwd: repo, env, stdio: ["ignore", "ignore", "ignore"], detached: true });
|
||||
backend.unref(); frontend.unref();
|
||||
await waitForHttp(`http://${HOST}:${BACKEND_PORT}/health`);
|
||||
await waitForHttp(`http://${HOST}:${FRONTEND_PORT}/`);
|
||||
await logHandle.close();
|
||||
owned.status = "RUNNING";
|
||||
owned.backend = { pid: backend.pid, port: BACKEND_PORT, command: [process.execPath, join(repo, "backend", "dist", "server.js")] };
|
||||
owned.frontend = { pid: frontend.pid, port: FRONTEND_PORT, command: [executables.pythonPath, "-m", "http.server", String(FRONTEND_PORT)] };
|
||||
await writeManualOwnership(root, owned);
|
||||
return owned;
|
||||
}
|
||||
async function processCommandMatches(pid, expectedCommand) {
|
||||
if (!Array.isArray(expectedCommand) || expectedCommand.length === 0) return false;
|
||||
let output;
|
||||
try {
|
||||
const { stdout } = await execFileAsync("ps", ["-p", String(pid), "-o", "command="], { encoding: "utf8" });
|
||||
output = stdout.trim();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (output.length === 0) return false;
|
||||
// The recorded command is the argv array used to spawn the process; verify every token appears
|
||||
// in the current command line in order, so a reused PID with unrelated command is refused.
|
||||
let cursor = 0;
|
||||
for (const token of expectedCommand) {
|
||||
if (token.length === 0) continue;
|
||||
const index = output.indexOf(token, cursor);
|
||||
if (index < 0) return false;
|
||||
cursor = index + token.length;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function stopManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repo });
|
||||
if (owned.status !== "RUNNING" || !owned.backend?.pid || !owned.frontend?.pid) throw new Error("manual acceptance is not running");
|
||||
for (const pid of [owned.backend.pid, owned.frontend.pid]) {
|
||||
try { process.kill(-pid, "SIGTERM"); } catch (error) { if (error?.code !== "ESRCH") throw error; }
|
||||
}
|
||||
const deadline = Date.now() + 15_000;
|
||||
while (Date.now() < deadline && (live(owned.backend.pid) || live(owned.frontend.pid))) await new Promise((resolvePromise) => setTimeout(resolvePromise, 250));
|
||||
owned.status = "STOPPED";
|
||||
await writeManualOwnership(root, owned);
|
||||
return owned;
|
||||
}
|
||||
|
||||
export async function cleanupManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repo });
|
||||
if (owned.status === "RUNNING") throw new Error("manual acceptance is still live");
|
||||
if (owned.backend?.pid && live(owned.backend.pid)) throw new Error("backend process is still live");
|
||||
if (owned.frontend?.pid && live(owned.frontend.pid)) throw new Error("frontend process is still live");
|
||||
const parent = dirname(root);
|
||||
const tombstone = join(parent, `.deleting-p11-${owned.nonce.slice(0, 16)}`);
|
||||
await rename(root, tombstone);
|
||||
await rm(tombstone, { recursive: true, force: false });
|
||||
}
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
if (argv.length !== 1 || !["prepare", "serve", "stop", "cleanup"].includes(argv[0])) throw new Error("usage: p11-manual-acceptance.mjs prepare|serve|stop|cleanup");
|
||||
switch (argv[0]) {
|
||||
case "prepare": await prepareManual(); break;
|
||||
case "serve": await serveManual(); break;
|
||||
case "stop": await stopManual(); break;
|
||||
case "cleanup": await cleanupManual(); break;
|
||||
}
|
||||
}
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) {
|
||||
try { await main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { access, lstat, readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import {
|
||||
cleanupManual,
|
||||
prepareManual,
|
||||
readManualOwnership,
|
||||
serveManual,
|
||||
stopManual,
|
||||
} from "./p11-manual-acceptance.mjs";
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const fixedRoot = join(repoRoot, ".artifacts", "manual-acceptance", "p11");
|
||||
|
||||
async function safeCleanup() {
|
||||
try {
|
||||
const owned = await readManualOwnership({ repositoryRoot: repoRoot });
|
||||
if (owned.status === "RUNNING") await stopManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
await cleanupManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
} catch {
|
||||
await rm(fixedRoot, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
await safeCleanup();
|
||||
});
|
||||
|
||||
test.afterEach(async () => {
|
||||
await safeCleanup();
|
||||
});
|
||||
|
||||
test("prepare creates an independent pending lab without verdict", { concurrency: false }, async () => {
|
||||
const root = await prepareManual({ repositoryRoot: repoRoot });
|
||||
assert.equal(root, fixedRoot);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repoRoot });
|
||||
assert.equal(owned.kind, "p11-manual-acceptance");
|
||||
assert.equal(owned.status, "PENDING");
|
||||
await access(join(root, "GUIDE.md"));
|
||||
await access(join(root, "author", "thoth-workspaces.yaml"));
|
||||
await access(join(root, "author", "p11-filesystem", "evidence", "guide.md"));
|
||||
await access(join(root, "requests", "validate-p11-filesystem.json"));
|
||||
await access(join(root, "commands", "http-01-status.sh"));
|
||||
await access(join(root, "commands", "render-1.sh"));
|
||||
await assert.rejects(access(join(root, "VERDICT.md")));
|
||||
const guide = await readFile(join(root, "GUIDE.md"), "utf8");
|
||||
assert.match(guide, /VERDICT\.md/);
|
||||
assert.match(guide, /read-only/);
|
||||
});
|
||||
|
||||
test("serve, stop, and cleanup manage the owned backend and frontend listeners", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
const running = await serveManual({ repositoryRoot: repoRoot });
|
||||
assert.equal(running.status, "RUNNING");
|
||||
assert.equal(typeof running.backend.pid, "number");
|
||||
assert.equal(typeof running.frontend.pid, "number");
|
||||
const status = await fetch("http://127.0.0.1:8791/workspace-registry/status");
|
||||
assert.equal(status.status, 200);
|
||||
const frontend = await fetch("http://127.0.0.1:8792/");
|
||||
assert.equal(frontend.status, 200);
|
||||
await assert.rejects(cleanupManual({ repositoryRoot: repoRoot }), /still live/);
|
||||
const stopped = await stopManual({ repositoryRoot: repoRoot });
|
||||
assert.equal(stopped.status, "STOPPED");
|
||||
await cleanupManual({ repositoryRoot: repoRoot });
|
||||
await assert.rejects(lstat(fixedRoot));
|
||||
});
|
||||
|
||||
test("stop fails closed when ownership is tampered", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
const running = await serveManual({ repositoryRoot: repoRoot });
|
||||
const ownershipPath = join(fixedRoot, "ownership.json");
|
||||
const digestPath = join(fixedRoot, "ownership.sha256");
|
||||
const original = JSON.parse(await readFile(ownershipPath, "utf8"));
|
||||
const tampered = { ...original, backend: { ...original.backend, pid: original.backend.pid + 1 } };
|
||||
await rm(ownershipPath);
|
||||
await readFile(join(fixedRoot, "logs", "backend.log"));
|
||||
await import("node:fs/promises").then(({ writeFile }) => writeFile(ownershipPath, `${JSON.stringify(tampered, null, 2)}
|
||||
`));
|
||||
await assert.rejects(stopManual({ repositoryRoot: repoRoot }), /manual ownership digest mismatch/);
|
||||
const restored = `${JSON.stringify(running, null, 2)}
|
||||
`;
|
||||
const restoredDigest = `${createHash("sha256").update(restored).digest("hex")}
|
||||
`;
|
||||
await import("node:fs/promises").then(({ writeFile }) => Promise.all([writeFile(ownershipPath, restored), writeFile(digestPath, restoredDigest)]));
|
||||
await stopManual({ repositoryRoot: repoRoot });
|
||||
});
|
||||
@@ -1,190 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants, lstatSync, realpathSync } from "node:fs";
|
||||
import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { ThtRunner } from "../dist/tht/tht-runner.js";
|
||||
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const HEX40 = /^[0-9a-f]{40}$/;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
|
||||
function fixedRoot(repositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p11"); }
|
||||
function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); }
|
||||
function assertNoSymlinks(root, path, allowMissingLeaf = false) {
|
||||
const rel = relative(root, path);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path is outside owned root");
|
||||
let cursor = root;
|
||||
const parts = rel.split(sep).filter(Boolean);
|
||||
for (const [index, part] of parts.entries()) {
|
||||
cursor = join(cursor, part);
|
||||
try { if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink"); }
|
||||
catch (error) {
|
||||
if (allowMissingLeaf && error?.code === "ENOENT" && index === parts.length - 1) return;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
async function ownership(repositoryRoot, ownershipPath) {
|
||||
const root = fixedRoot(repositoryRoot);
|
||||
const expected = join(root, "ownership.json");
|
||||
if (resolve(ownershipPath) !== expected) throw new Error("ownership path is not owned");
|
||||
const rootEntry = await lstat(root); const ownershipEntry = await lstat(expected);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership is unsafe");
|
||||
if (await realpath(root) !== root) throw new Error("ownership root is not canonical");
|
||||
let value; try { value = JSON.parse(await readFile(expected, "utf8")); } catch { throw new Error("ownership is malformed"); }
|
||||
if (value?.schemaVersion !== 1 || value.kind !== "p11-manual-acceptance" || !HEX64.test(value.nonce ?? "") || value.root !== root || value.repositoryRoot !== realpathSync(repositoryRoot)) {
|
||||
throw new Error("ownership identity mismatch");
|
||||
}
|
||||
return { root, value };
|
||||
}
|
||||
const ANCHORED_PUBLISH_SOURCE=String.raw`import os,secrets,stat,sys
|
||||
parent,name,expected_dev,expected_ino=sys.argv[1:]
|
||||
pfd=fd=None;stage=".render-stage-"+secrets.token_hex(16);published=False
|
||||
def fail(): raise RuntimeError("anchored publication refused")
|
||||
try:
|
||||
pfd=os.open(parent,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
||||
identity=os.fstat(pfd)
|
||||
if (identity.st_dev,identity.st_ino)!=(int(expected_dev),int(expected_ino)): fail()
|
||||
try: os.stat(name,dir_fd=pfd,follow_symlinks=False); fail()
|
||||
except FileNotFoundError: pass
|
||||
fd=os.open(stage,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=pfd)
|
||||
data=sys.stdin.buffer.read(33554433)
|
||||
if len(data)>33554432: fail()
|
||||
view=memoryview(data)
|
||||
while view:
|
||||
written=os.write(fd,view)
|
||||
if written<=0: fail()
|
||||
view=view[written:]
|
||||
os.fsync(fd);os.close(fd);fd=None;os.rename(stage,name,src_dir_fd=pfd,dst_dir_fd=pfd);published=True;os.fsync(pfd)
|
||||
current=os.stat(parent,follow_symlinks=False)
|
||||
if not stat.S_ISDIR(current.st_mode) or (current.st_dev,current.st_ino)!=(identity.st_dev,identity.st_ino): fail()
|
||||
except Exception:
|
||||
if published:
|
||||
try: os.unlink(name,dir_fd=pfd);os.fsync(pfd)
|
||||
except Exception: pass
|
||||
print("anchored output publication refused (details redacted)",file=sys.stderr);raise SystemExit(1)
|
||||
finally:
|
||||
if fd is not None: os.close(fd)
|
||||
if pfd is not None:
|
||||
try: os.unlink(stage,dir_fd=pfd)
|
||||
except FileNotFoundError: pass
|
||||
os.close(pfd)
|
||||
`;
|
||||
async function atomicCopy(source, output) {
|
||||
const parent = dirname(output);
|
||||
const entry = await lstat(parent);
|
||||
if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("rendered parent identity is unsafe");
|
||||
const bytes = await readFile(source);
|
||||
const result = spawnSync("python3", ["-c", ANCHORED_PUBLISH_SOURCE, parent, basename(output), String(entry.dev), String(entry.ino)], { input: bytes, encoding: "utf8", maxBuffer: 1024 * 1024 });
|
||||
if (result.error || result.status !== 0) throw new Error("anchored output publication refused; rendered parent identity changed or output is unsafe");
|
||||
}
|
||||
function sameEntry(actual, expected) { return actual.dev === expected.dev && actual.ino === expected.ino; }
|
||||
async function readBounded(path, max, label) {
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const before = await handle.stat(); const pathEntry = await lstat(path);
|
||||
if (!before.isFile() || pathEntry.isSymbolicLink() || !pathEntry.isFile() || !sameEntry(before, pathEntry)) throw new Error(`${label} is unsafe`);
|
||||
if (before.size < 1 || before.size > max) throw new Error(`${label} is unbounded`);
|
||||
const bytes = Buffer.alloc(before.size); let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset);
|
||||
if (bytesRead < 1) throw new Error(`${label} changed while reading`);
|
||||
offset += bytesRead;
|
||||
}
|
||||
const after = await handle.stat();
|
||||
if (!sameEntry(before, after) || after.size !== before.size) throw new Error(`${label} changed while reading`);
|
||||
return bytes;
|
||||
} finally {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
async function readSnapshotManifest(root, manifestPath, commit, yamlName, expectedDigest) {
|
||||
let manifestEntry;
|
||||
try { assertNoSymlinks(root, manifestPath); manifestEntry = await lstat(manifestPath); }
|
||||
catch (error) { if (error?.code === "ENOENT") throw new Error("snapshot manifest is missing or unbounded"); throw error; }
|
||||
if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink() || await realpath(manifestPath) !== manifestPath) throw new Error("snapshot manifest is unsafe");
|
||||
const bytes = await readBounded(manifestPath, 1024 * 1024, "snapshot manifest");
|
||||
let manifest; try { manifest = JSON.parse(bytes.toString("utf8")); } catch { throw new Error("snapshot manifest is malformed"); }
|
||||
const files = manifest?.files;
|
||||
if (manifest?.head !== commit || !files || typeof files !== "object" || Array.isArray(files)) throw new Error("snapshot manifest identity is unsafe");
|
||||
if (!HEX64.test(files[yamlName] ?? "") || files[yamlName] !== expectedDigest) throw new Error("snapshot manifest digest is unsafe");
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export async function renderOwnedSnapshot({ repositoryRoot = defaultRepositoryRoot, ownershipPath, snapshotPath, outputPath, snapshotSha256, env = process.env, beforePublish }) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const { root } = await ownership(repo, resolve(repo, ownershipPath));
|
||||
const snapshot = resolve(repo, snapshotPath);
|
||||
const output = resolve(repo, outputPath);
|
||||
const snapshotsRoot = join(root, "installation", "registry", "snapshots");
|
||||
const renderedRoot = join(root, "rendered");
|
||||
if (!isAbsolute(snapshotPath) || !below(snapshotsRoot, snapshot)) throw new Error("snapshot is not an owned absolute path");
|
||||
const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(relative(snapshotsRoot, snapshot).split(sep).join("/"));
|
||||
if (!match || !HEX40.test(match[1])) throw new Error("snapshot is not commit addressed");
|
||||
if (!HEX64.test(snapshotSha256 ?? "")) throw new Error("snapshot digest identity is unsafe");
|
||||
assertNoSymlinks(root, snapshot);
|
||||
const snapshotEntry = await lstat(snapshot);
|
||||
if (!snapshotEntry.isFile() || snapshotEntry.isSymbolicLink() || await realpath(snapshot) !== snapshot) throw new Error("snapshot is unsafe");
|
||||
const yamlName = `${match[2]}.yaml`;
|
||||
await readSnapshotManifest(root, join(snapshotsRoot, match[1], "snapshot.json"), match[1], yamlName, snapshotSha256);
|
||||
const snapshotBytes = await readBounded(snapshot, 1024 * 1024, "snapshot");
|
||||
if (createHash("sha256").update(snapshotBytes).digest("hex") !== snapshotSha256) throw new Error("snapshot bytes changed");
|
||||
if (!below(renderedRoot, output) || dirname(output) !== renderedRoot || !output.endsWith(".yaml")) throw new Error("output is not an owned rendered path");
|
||||
assertNoSymlinks(root, dirname(output));
|
||||
try { if ((await lstat(output)).isSymbolicLink()) throw new Error("output is unsafe"); } catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
await mkdir(join(snapshotsRoot, "runtime"), { recursive: true, mode: 0o700 });
|
||||
const bindingEnv = Object.fromEntries((await readFile(join(root, "installation", "bindings.env"), "utf8")).trim().split(/\n+/).filter(Boolean).map((line) => line.split(/=(.+)/)));
|
||||
const effectiveEnv = { ...bindingEnv, ...env };
|
||||
const prior = {};
|
||||
for (const [key, value] of Object.entries(effectiveEnv)) { prior[key] = process.env[key]; if (value === undefined) delete process.env[key]; else process.env[key] = value; }
|
||||
const runner = new ThtRunner({
|
||||
thtBin: join(repo, "harness", ".venv", "bin", "tht"),
|
||||
harnessDir: join(repo, "harness"),
|
||||
configPath: join(root, "installation", "runtime", "base.yaml"),
|
||||
dataRoot: join(root, "installation", "data"),
|
||||
runtimeSnapshotRoot: join(snapshotsRoot, "runtime"),
|
||||
secretRoots: [join(root, "fixture-secrets")],
|
||||
semanticRuntime: { internalQdrantUrl: "http://qdrant:6333", internalEmbeddingUrl: "http://embedding:11434", internalEmbeddingModel: "qwen3-embedding:0.6b", internalEmbeddingDimensions: 1024 },
|
||||
});
|
||||
let lease;
|
||||
try {
|
||||
lease = runner.acquireWorkspaceRuntime(snapshot);
|
||||
const verifySnapshot = async () => {
|
||||
const current = await readBounded(snapshot, 1024 * 1024, "snapshot");
|
||||
if (createHash("sha256").update(current).digest("hex") !== snapshotSha256) throw new Error("snapshot content changed during rendering");
|
||||
};
|
||||
await verifySnapshot();
|
||||
if (beforePublish) await beforePublish({ output, renderedRoot });
|
||||
await verifySnapshot();
|
||||
await atomicCopy(lease.path, output);
|
||||
} finally {
|
||||
if (lease) lease.release();
|
||||
for (const key of Object.keys(env)) { if (prior[key] === undefined) delete process.env[key]; else process.env[key] = prior[key]; }
|
||||
}
|
||||
return output;
|
||||
}
|
||||
function parseArgs(argv) {
|
||||
if (argv.length !== 8) throw new Error("usage: p11-render-snapshot.mjs --ownership PATH --snapshot ABSOLUTE_PATH --output PATH --snapshot-sha256 HEX");
|
||||
const result = {};
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
if (!["--ownership", "--snapshot", "--output", "--snapshot-sha256"].includes(argv[index]) || result[argv[index]]) throw new Error("invalid arguments");
|
||||
result[argv[index]] = argv[index + 1];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) {
|
||||
try {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
await renderOwnedSnapshot({ ownershipPath: args["--ownership"], snapshotPath: args["--snapshot"], outputPath: args["--output"], snapshotSha256: args["--snapshot-sha256"] });
|
||||
console.log(`rendered ${resolve(args["--output"])}`);
|
||||
} catch (error) {
|
||||
console.error(`p11 render refused: ${error.message}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { access, readFile, rm } from "node:fs/promises";
|
||||
import { join, dirname, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { renderOwnedSnapshot } from "./p11-render-snapshot.mjs";
|
||||
import { cleanupManual, prepareManual, readManualOwnership, serveManual, stopManual } from "./p11-manual-acceptance.mjs";
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const fixedRoot = join(repoRoot, ".artifacts", "manual-acceptance", "p11");
|
||||
|
||||
async function safeCleanup() {
|
||||
try {
|
||||
const owned = await readManualOwnership({ repositoryRoot: repoRoot });
|
||||
if (owned.status === "RUNNING") await stopManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
await cleanupManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
} catch {
|
||||
await rm(fixedRoot, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
test.beforeEach(async () => { await safeCleanup(); });
|
||||
test.afterEach(async () => { await safeCleanup(); });
|
||||
|
||||
test("renderer rejects unowned ownership and out-of-root snapshot paths", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
const outside = join(repoRoot, "outside.yaml");
|
||||
await import("node:fs/promises").then(({ writeFile }) => writeFile(outside, "x"));
|
||||
await assert.rejects(renderOwnedSnapshot({
|
||||
repositoryRoot: repoRoot,
|
||||
ownershipPath: join(repoRoot, "ownership.json"),
|
||||
snapshotPath: outside,
|
||||
outputPath: join(fixedRoot, "rendered", "bad.yaml"),
|
||||
snapshotSha256: "a".repeat(64),
|
||||
}));
|
||||
await rm(outside, { force: true });
|
||||
});
|
||||
|
||||
test("renderer copies an owned runtime lease deterministically", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
await serveManual({ repositoryRoot: repoRoot });
|
||||
const validateRequest = JSON.parse(await readFile(join(fixedRoot, "requests", "validate-p11-filesystem.json"), "utf8"));
|
||||
const status = await fetch("http://127.0.0.1:8791/workspace-registry/status");
|
||||
const statusBody = await status.json();
|
||||
const publish = await fetch("http://127.0.0.1:8791/workspaces/publish", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ action: "create", workspace: validateRequest.workspace, baseCommit: statusBody.head }),
|
||||
});
|
||||
assert.equal(publish.status, 200);
|
||||
const readResponse = await fetch("http://127.0.0.1:8791/workspaces/p11-filesystem");
|
||||
const readBody = await readResponse.json();
|
||||
const snapshotPath = readBody.revision.snapshotPath;
|
||||
const manifest = JSON.parse(await readFile(join(dirname(snapshotPath), "snapshot.json"), "utf8"));
|
||||
const digest = manifest.files["p11-filesystem.yaml"];
|
||||
const one = join(fixedRoot, "rendered", "one.yaml");
|
||||
const two = join(fixedRoot, "rendered", "two.yaml");
|
||||
await renderOwnedSnapshot({ repositoryRoot: repoRoot, ownershipPath: join(fixedRoot, "ownership.json"), snapshotPath, outputPath: one, snapshotSha256: digest });
|
||||
await renderOwnedSnapshot({ repositoryRoot: repoRoot, ownershipPath: join(fixedRoot, "ownership.json"), snapshotPath, outputPath: two, snapshotSha256: digest });
|
||||
assert.equal(await readFile(one, "utf8"), await readFile(two, "utf8"));
|
||||
await access(one);
|
||||
await access(two);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,158 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p2-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p2-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p11-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p2 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p2-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p11-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p2-${"A".repeat(32)}`), `p2-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p11-integration", `p11-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p2-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p2 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p2-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p2 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p2-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p2-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p11-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p2-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P2_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P2_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P2_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P2 automated integration: PASS/);
|
||||
assert.match(reportMd, /P2 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P2_ACCEPTANCE_SYNTHETIC: "1", P2_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p2-acceptance/);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,158 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p2p6-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p2p6-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p2p6-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p2p6 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p2p6-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p2p6-${"A".repeat(32)}`), `p2p6-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p2p6-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p2p6 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p2p6-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p2p6 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p2p6-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p2p6-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p2p6-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P2P6_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P2P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P2P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P2P6 automated integration: PASS/);
|
||||
assert.match(reportMd, /P2P6 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P2P6_ACCEPTANCE_SYNTHETIC: "1", P2P6_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p2p6-acceptance/);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,158 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p3-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p3-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p3-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p3 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p3-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p3-${"A".repeat(32)}`), `p3-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p3-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p3 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p3-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p3 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p3-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p3-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p3-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P3_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P3_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P3_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P3 automated integration: PASS/);
|
||||
assert.match(reportMd, /P3 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P3_ACCEPTANCE_SYNTHETIC: "1", P3_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p3-acceptance/);
|
||||
});
|
||||
@@ -1,403 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// P4 automated integration acceptance: Qdrant collection lifecycle (self-heal + guarded rebuild).
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { execFile, execFileSync } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { createServer as createNetServer } from "node:net";
|
||||
import process from "node:process";
|
||||
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
|
||||
import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const RUN_ID = /^p4-[0-9a-f]{32}$/;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
const QDRANT_IMAGE = "qdrant/qdrant:v1.18.2";
|
||||
export const CHECK_IDS = Object.freeze([
|
||||
"preflight",
|
||||
"clean_state",
|
||||
"ownership",
|
||||
"qdrant_up",
|
||||
"self_heal_create_missing",
|
||||
"self_heal_repairs_missing_index",
|
||||
"incompatible_refused",
|
||||
"require_existing_refused",
|
||||
"rebuild_recreates_contract",
|
||||
"secret_scan",
|
||||
"cleanup_confinement",
|
||||
]);
|
||||
const TOPOLOGY = ["installation", "fixtures", "logs", "qdrant-volumes"];
|
||||
const MAX_REPORT_JSON_BYTES = 64 * 1024;
|
||||
const MAX_REPORT_MD_BYTES = 32 * 1024;
|
||||
|
||||
|
||||
function resolveSystemExecutable(name) {
|
||||
for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`, `/usr/local/sbin/${name}`]) {
|
||||
try {
|
||||
const resolved = realpathSync(candidate);
|
||||
if (statSync(resolved).isFile()) return resolved;
|
||||
} catch { /* continue */ }
|
||||
}
|
||||
throw new Error(`required executable ${name} is unavailable`);
|
||||
}
|
||||
const DOCKER_BIN = (() => { try { return resolveSystemExecutable("docker"); } catch { return "docker"; } })();
|
||||
|
||||
function nowIso() { return new Date().toISOString(); }
|
||||
function sha256(value) { return createHash("sha256").update(value).digest("hex"); }
|
||||
function assert(condition, message) { if (!condition) throw new Error(message); }
|
||||
function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
|
||||
|
||||
function canonicalRoot(repositoryRoot = defaultRepositoryRoot) {
|
||||
return realpathSync(repositoryRoot);
|
||||
}
|
||||
export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) {
|
||||
return join(canonicalRoot(repositoryRoot), ".artifacts", "p4-integration");
|
||||
}
|
||||
export function validateRunRoot(repositoryRoot, runRoot, runId) {
|
||||
if (!RUN_ID.test(runId)) throw new Error("invalid owned run id");
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const lexical = resolve(runRoot);
|
||||
if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child");
|
||||
return lexical;
|
||||
}
|
||||
function validateNoSymlinkAncestors(repositoryRoot, target) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const rel = relative(repo, target);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("target escapes the repository");
|
||||
let cursor = repo;
|
||||
for (const part of rel.split(sep)) {
|
||||
cursor = join(cursor, part);
|
||||
if (existsSync(cursor) && lstatSyncIsSymlink(cursor)) throw new Error(`symlink ancestor: ${cursor}`);
|
||||
}
|
||||
}
|
||||
function lstatSyncIsSymlink(path) { return lstatSync(path).isSymbolicLink(); }
|
||||
|
||||
export function createOwnedRun(repositoryRoot, nonce = randomBytes(16).toString("hex")) {
|
||||
const runId = `p4-${nonce}`;
|
||||
if (!RUN_ID.test(runId)) throw new Error("invalid run id");
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
mkdirSync(base, { recursive: true });
|
||||
const runRoot = join(base, runId);
|
||||
validateNoSymlinkAncestors(repositoryRoot, runRoot);
|
||||
mkdirSync(join(runRoot, "installation"), { recursive: true });
|
||||
mkdirSync(join(runRoot, "fixtures"), { recursive: true });
|
||||
mkdirSync(join(runRoot, "logs"), { recursive: true });
|
||||
mkdirSync(join(runRoot, "qdrant-volumes"), { recursive: true });
|
||||
const marker = { runId, createdAt: nowIso(), repositoryRoot: canonicalRoot(repositoryRoot), sha256: "" };
|
||||
marker.sha256 = sha256(JSON.stringify(marker) + "\n");
|
||||
writeFileSync(join(runRoot, "run.json"), JSON.stringify(marker, null, 2) + "\n", { mode: 0o600 });
|
||||
return { runId, runRoot };
|
||||
}
|
||||
|
||||
export function cleanupOwnedRun(repositoryRoot, runRoot, runId) {
|
||||
const validated = validateRunRoot(repositoryRoot, runRoot, runId);
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
for (const sibling of readdirSync(base)) {
|
||||
if (sibling.startsWith("p4-") && sibling !== runId) throw new Error("refusing cleanup with sibling p4 runs present");
|
||||
}
|
||||
rmSync(validated, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
|
||||
function result(checkId, ok, detail, cause) {
|
||||
const message = cause ? `${String(detail)} :: ${String(cause)}` : String(detail);
|
||||
return { checkId, status: ok ? "PASS" : "FAIL", ok: !!ok, detail: ok ? "PASS" : message.slice(0, 500) };
|
||||
}
|
||||
|
||||
function execCapture(command, args, options = {}) {
|
||||
const spawned = execFileSync(command, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, ...options });
|
||||
return String(spawned ?? "");
|
||||
}
|
||||
|
||||
async function waitForQdrant(baseUrl, timeoutMs = 120000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/readyz`, { signal: AbortSignal.timeout(3000) });
|
||||
if (res.ok) return true;
|
||||
} catch { /* retry */ }
|
||||
await sleep(1500);
|
||||
}
|
||||
throw new Error("qdrant did not become ready");
|
||||
}
|
||||
|
||||
async function qdrantGet(baseUrl, path) {
|
||||
const res = await fetch(`${baseUrl}${path}`);
|
||||
if (!res.ok) throw new Error(`qdrant GET ${path} -> ${res.status}`);
|
||||
return (await res.json()).result;
|
||||
}
|
||||
async function qdrantPut(baseUrl, path, body) {
|
||||
const payload = { ...body };
|
||||
if (payload.vectors && typeof payload.vectors.distance === "string" && payload.vectors.distance.length > 0) {
|
||||
payload.vectors = { ...payload.vectors, distance: payload.vectors.distance.charAt(0).toUpperCase() + payload.vectors.distance.slice(1) };
|
||||
}
|
||||
const res = await fetch(`${baseUrl}${path}`, {
|
||||
method: "PUT",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (!res.ok && res.status !== 409) throw new Error(`qdrant PUT ${path} -> ${res.status}`);
|
||||
return res.ok || res.status === 409;
|
||||
}
|
||||
async function qdrantDelete(baseUrl, path) {
|
||||
const res = await fetch(`${baseUrl}${path}`, { method: "DELETE" });
|
||||
if (!res.ok && res.status !== 404) throw new Error(`qdrant DELETE ${path} -> ${res.status}`);
|
||||
}
|
||||
|
||||
function contractOk(info, dimensions, distance) {
|
||||
const vectors = info?.config?.params?.vectors;
|
||||
const schema = info?.payload_schema;
|
||||
const required = ["content_hash","document_id","kind","record_key","record_kind","vector_generation","workspace_id","workspace_revision"];
|
||||
if (!vectors || vectors.size !== dimensions || String(vectors.distance).toLowerCase() !== distance) return false;
|
||||
if (!schema || typeof schema !== "object") return false;
|
||||
return required.every((field) => schema[field]?.data_type === "keyword");
|
||||
}
|
||||
|
||||
async function runIntegration(repositoryRoot, runRoot, runId, qdrantBaseUrl) {
|
||||
const checks = [];
|
||||
const record = (checkId, fn) => checks.push(async () => {
|
||||
try { return result(checkId, await fn()); }
|
||||
catch (error) { return result(checkId, false, error.message, error.cause?.message ?? error.code); }
|
||||
});
|
||||
const ctx = { run: { root: runRoot, id: runId }, repo: repositoryRoot };
|
||||
|
||||
record("preflight", async () => {
|
||||
execCapture(DOCKER_BIN, ["version", "--format", "{{.Server.Version}}"]);
|
||||
execCapture("node", ["--version"]);
|
||||
execCapture("npm", ["--version"]);
|
||||
return true;
|
||||
});
|
||||
|
||||
record("clean_state", async () => {
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const leftovers = readdirSync(base).filter((entry) => entry.startsWith("p4-") && entry !== runId);
|
||||
if (leftovers.length > 0) throw new Error(`leftover p4 runs: ${leftovers.join(", ")}`);
|
||||
return true;
|
||||
});
|
||||
|
||||
record("ownership", async () => {
|
||||
const marker = JSON.parse(await readFile(join(runRoot, "run.json"), "utf8"));
|
||||
if (marker.runId !== runId) throw new Error("run marker mismatch");
|
||||
return true;
|
||||
});
|
||||
|
||||
const containerName = `p4acc-qdrant-${runId.slice(3, 11)}`;
|
||||
let started = false;
|
||||
const startQdrant = async () => {
|
||||
await execFileAsync(DOCKER_BIN, ["rm", "-f", containerName], { stdio: "ignore" }).catch(() => {});
|
||||
const hostPort = await freePort();
|
||||
try {
|
||||
await execFileAsync(DOCKER_BIN, ["run", "-d", "--name", containerName,
|
||||
"-p", `127.0.0.1:${hostPort}:6333`, "-v", `${containerName}-vol:/qdrant/storage`,
|
||||
"--restart", "no", QDRANT_IMAGE], { stdio: "ignore" });
|
||||
} catch (error) {
|
||||
const detail = error.stderr ?? error.message;
|
||||
throw new Error(`docker run qdrant failed: ${String(detail).slice(0, 300)}`);
|
||||
}
|
||||
started = true;
|
||||
return `http://127.0.0.1:${hostPort}`;
|
||||
};
|
||||
const stopQdrant = async () => {
|
||||
if (!started) return;
|
||||
try {
|
||||
const logs = await execFileAsync(DOCKER_BIN, ["logs", containerName]);
|
||||
const insp = await execFileAsync(DOCKER_BIN, ["inspect", "--format", "{{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}", containerName]).catch(() => ({ stdout: "inspect failed" }));
|
||||
await writeFile(join(runRoot, "qdrant.log"), `INSPECT: ${String(insp.stdout).trim()}\n` + String(logs.stdout).slice(-3000) + "\n---STDERR---\n" + String(logs.stderr).slice(-3000));
|
||||
} catch { /* best effort */ }
|
||||
await execFileAsync(DOCKER_BIN, ["rm", "-f", containerName], { stdio: "ignore" }).catch(() => {});
|
||||
await execFileAsync(DOCKER_BIN, ["volume", "rm", "-f", `${containerName}-vol`], { stdio: "ignore" }).catch(() => {});
|
||||
};
|
||||
|
||||
|
||||
|
||||
function freePort() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createNetServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const port = server.address().port;
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function dockerPortRetry(containerName, attempts = 20) {
|
||||
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
||||
try {
|
||||
const inspect = await execFileAsync(DOCKER_BIN, ["port", containerName, "6333"]);
|
||||
const line = String(inspect.stdout).trim();
|
||||
const hostPort = line.split("\n")[0].split(":")[1];
|
||||
if (hostPort) return `http://127.0.0.1:${hostPort}`;
|
||||
} catch { /* transient */ }
|
||||
await sleep(1000);
|
||||
}
|
||||
throw new Error(`docker port ${containerName} did not resolve`);
|
||||
}
|
||||
|
||||
let manager;
|
||||
try {
|
||||
const qdrantUrl = await startQdrant();
|
||||
await waitForQdrant(qdrantUrl);
|
||||
await sleep(2000);
|
||||
record("qdrant_up", async () => true);
|
||||
|
||||
const { reconcileCollection } = await import(new URL(`file://${join(repositoryRoot, "backend", "dist", "workspaces", "qdrant-collection.js")}`).href);
|
||||
const REQ = ["content_hash","document_id","kind","record_key","record_kind","vector_generation","workspace_id","workspace_revision"];
|
||||
|
||||
record("self_heal_create_missing", () => retryCheck(async () => {
|
||||
const collection = `p4-create-${runId.slice(3, 11)}`;
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (!outcome.ok) throw new Error(`unexpected ${outcome.code}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (!contractOk(info, 1024, "cosine")) throw new Error("created contract mismatch");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("self_heal_repairs_missing_index", () => retryCheck(async () => {
|
||||
const collection = `p4-repair-${runId.slice(3, 11)}`;
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } });
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (outcome.ok !== true || outcome.state !== "repaired") throw new Error(`expected repaired, got ${JSON.stringify(outcome)}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (!contractOk(info, 1024, "cosine")) throw new Error("repaired contract mismatch");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("incompatible_refused", () => retryCheck(async () => {
|
||||
const collection = `p4-bad-${runId.slice(3, 11)}`;
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 768, distance: "cosine" } });
|
||||
const before = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (outcome.ok !== false || outcome.code !== "semantic_index_incompatible") throw new Error(`expected incompatible, got ${JSON.stringify(outcome)}`);
|
||||
const after = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (JSON.stringify(before) !== JSON.stringify(after)) throw new Error("incompatible collection was mutated");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("require_existing_refused", () => retryCheck(async () => {
|
||||
const collection = `p4-missing-${runId.slice(3, 11)}`;
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "require_existing" });
|
||||
if (outcome.ok !== false || outcome.code !== "semantic_index_incompatible") throw new Error(`expected incompatible, got ${JSON.stringify(outcome)}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`).catch(() => undefined);
|
||||
if (info !== undefined) throw new Error("require_existing created a collection");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("rebuild_recreates_contract", () => retryCheck(async () => {
|
||||
const collection = `p4-rebuild-${runId.slice(3, 11)}`;
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } });
|
||||
await qdrantDelete(qdrantUrl, `/collections/${collection}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`).catch(() => undefined);
|
||||
if (info !== undefined) throw new Error("rebuild did not delete the collection");
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } });
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (!outcome.ok) throw new Error(`recreate verify failed ${JSON.stringify(outcome)}`);
|
||||
const recreated = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (!contractOk(recreated, 1024, "cosine")) throw new Error("recreated contract mismatch");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("secret_scan", async () => {
|
||||
const secretValues = ["p4-acceptance"];
|
||||
const findings = await scanSecrets({ runRoot, forbiddenValues: secretValues, expectedGitRepositories: [] });
|
||||
if (findings.length > 0) throw new Error(`secret findings: ${findings.join(", ")}`);
|
||||
return true;
|
||||
});
|
||||
|
||||
record("cleanup_confinement", async () => {
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const direct = readdirSync(base).filter((entry) => entry.startsWith("p4-"));
|
||||
if (direct.length !== 1 || direct[0] !== runId) throw new Error("run confinement violated");
|
||||
return true;
|
||||
});
|
||||
const settledChecks = await runChecks(checks);
|
||||
return settledChecks;
|
||||
} finally {
|
||||
await stopQdrant();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async function retryCheck(fn, attempts = 3) {
|
||||
let lastError;
|
||||
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
||||
try { return await fn(); } catch (error) { lastError = error; await sleep(3000); }
|
||||
}
|
||||
try {
|
||||
const ps = await execFileAsync(DOCKER_BIN, ["ps", "-a", "--filter", "name=p4acc-qdrant", "--format", "{{.Names}} {{.Status}} {{.Ports}}"]);
|
||||
lastError = new Error(`${lastError.message} | containers: ${String(ps.stdout).trim()}`);
|
||||
} catch { /* best effort */ }
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
async function runChecks(checks) {
|
||||
const settled = [];
|
||||
for (const check of checks) settled.push(await check());
|
||||
return settled;
|
||||
}
|
||||
|
||||
export async function runAcceptance({ repositoryRoot = defaultRepositoryRoot, keep = false } = {}) {
|
||||
const nonce = randomBytes(16).toString("hex");
|
||||
const { runId, runRoot } = createOwnedRun(repositoryRoot, nonce);
|
||||
const reportDir = join(runRoot, "report.md");
|
||||
const reportJsonDir = join(runRoot, "report.json");
|
||||
try {
|
||||
await execFileAsync("npm", ["--prefix", join(repositoryRoot, "backend"), "run", "build"], { stdio: "ignore" });
|
||||
const checks = await runIntegration(repositoryRoot, runRoot, runId, "");
|
||||
const overall = deriveOverall(checks);
|
||||
const summary = {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
phase: "p4",
|
||||
checks,
|
||||
overall,
|
||||
boundCommit: execCapture("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot }).trim(),
|
||||
};
|
||||
await writeFile(reportJsonDir, JSON.stringify(summary, null, 2) + "\n");
|
||||
const rows = checks.map((c) => `- [${c.ok ? "x" : " "}] ${c.checkId}: ${c.detail}`).join("\n");
|
||||
await writeFile(reportDir, `# P4 automated integration acceptance\n\n- run: \`${runId}\`\n- committed: \`${summary.boundCommit}\`\n\n${rows}\n\n**Overall: ${overall}**\n`);
|
||||
if (overall === "PASS") {
|
||||
if (!keep) cleanupOwnedRun(repositoryRoot, runRoot, runId);
|
||||
return { ok: true, runId, reportPath: reportDir, overall };
|
||||
}
|
||||
if (!keep) {
|
||||
try {
|
||||
const validated = validateRunRoot(repositoryRoot, runRoot, runId);
|
||||
rmSync(validated, { recursive: true, force: true });
|
||||
} catch { /* best effort */ }
|
||||
}
|
||||
return { ok: false, runId, reportPath: reportDir, overall };
|
||||
} catch (error) {
|
||||
try {
|
||||
const partial = { schemaVersion: 1, runId, phase: "p4", checks: [], overall: "FAIL", error: String(error).slice(0, 500) };
|
||||
await writeFile(reportJsonDir, JSON.stringify(partial, null, 2) + "\n");
|
||||
await writeFile(reportDir, `# P4 automated integration acceptance\n\n- run: \`${runId}\`\n- error: \`${String(error).slice(0, 500)}\`\n\n**Overall: FAIL**\n`);
|
||||
} catch { /* best effort */ }
|
||||
if (keep) return { ok: false, runId, reportPath: reportDir, overall: "FAIL" };
|
||||
try {
|
||||
const validated = validateRunRoot(repositoryRoot, runRoot, runId);
|
||||
rmSync(validated, { recursive: true, force: true });
|
||||
} catch { /* best effort */ }
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const keep = args.includes("--keep");
|
||||
runAcceptance({ keep }).then((outcome) => {
|
||||
process.stdout.write(`P4 automated integration: ${outcome.overall}\nrun: ${outcome.runId}\nreport: ${outcome.reportPath}\n`);
|
||||
process.exit(outcome.ok ? 0 : 1);
|
||||
}).catch((error) => {
|
||||
process.stderr.write(`P4 automated integration: FAIL\n${String(error)}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
validateRunRoot,
|
||||
} from "./p4-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p4-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p4-integration"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("check ids are stable and unique", () => {
|
||||
assert.equal(new Set(CHECK_IDS).size, CHECK_IDS.length);
|
||||
assert.ok(CHECK_IDS.includes("self_heal_create_missing"));
|
||||
assert.ok(CHECK_IDS.includes("rebuild_recreates_contract"));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p4 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p4-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [base, join(repositoryRoot, ".artifacts", "p1-integration", id), join(base, id, "nested")]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p4-${"A".repeat(32)}`), `p4-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("createOwnedRun writes a canonical marker and cleanup refuses foreign roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const { runId, runRoot } = createOwnedRun(repositoryRoot);
|
||||
assert.match(runId, /^p4-[0-9a-f]{32}$/);
|
||||
const marker = JSON.parse(await readFile(join(runRoot, "run.json"), "utf8"));
|
||||
assert.equal(marker.runId, runId);
|
||||
assert.throws(() => cleanupOwnedRun(repositoryRoot, join(repositoryRoot, "tmp"), runId));
|
||||
cleanupOwnedRun(repositoryRoot, runRoot, runId);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,158 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p5-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p5-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p5-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p5 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p5-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p5-${"A".repeat(32)}`), `p5-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p5-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p5 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p5-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p5 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p5-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p5-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p5-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P5_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P5_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P5_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P5 automated integration: PASS/);
|
||||
assert.match(reportMd, /P5 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P5_ACCEPTANCE_SYNTHETIC: "1", P5_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p5-acceptance/);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,158 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p6-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p6-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p6-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p6 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p6-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p6-${"A".repeat(32)}`), `p6-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p6-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p6 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p6-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p6 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p6-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p6-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p6-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P6_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P6 automated integration: PASS/);
|
||||
assert.match(reportMd, /P6 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P6_ACCEPTANCE_SYNTHETIC: "1", P6_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p6-acceptance/);
|
||||
});
|
||||
@@ -1,943 +0,0 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import ts from "typescript";
|
||||
import { literalBashHeredocBodyRanges } from "./bash-heredoc.mjs";
|
||||
import { isMap, isScalar, isSeq, parseAllDocuments } from "yaml";
|
||||
|
||||
|
||||
/**
|
||||
* Revision-state absence policy by source dialect.
|
||||
* JS/TS syntax uses the TypeScript parser and YAML structure uses the installed YAML parser.
|
||||
* Shell active consumers are executable code/expansions and jq filter arguments for bare or
|
||||
* path-qualified jq, optionally through command or env. Quoted heredoc bodies are literal.
|
||||
* PowerShell analyzes executable code and nested $() in expandable strings. Python policy is
|
||||
* batched through the isolated stdlib AST helper. jq filters use a bounded path lexer after
|
||||
* shell argv/wrapper resolution. Offset-preserving transformations keep AST spans stable.
|
||||
*/
|
||||
const revisionIdentifiers = new Set(["revision", "workspaceRevision", "selectedWorkspace"]);
|
||||
|
||||
function unwrapExpression(node) {
|
||||
let current = node;
|
||||
while (ts.isParenthesizedExpression(current) || ts.isAsExpression(current) ||
|
||||
ts.isTypeAssertionExpression(current) || ts.isNonNullExpression(current) ||
|
||||
ts.isSatisfiesExpression(current)) {
|
||||
current = current.expression;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function isRevisionName(value, caseInsensitive) {
|
||||
if (typeof value !== "string") return false;
|
||||
if (!caseInsensitive) return revisionIdentifiers.has(value);
|
||||
const lower = value.toLowerCase();
|
||||
return lower === "revision" || lower === "workspacerevision" || lower === "selectedworkspace";
|
||||
}
|
||||
|
||||
function isRevisionExpression(node, caseInsensitive = false) {
|
||||
const unwrapped = unwrapExpression(node);
|
||||
if (ts.isIdentifier(unwrapped)) {
|
||||
const normalized = unwrapped.text.startsWith("$") && !unwrapped.text.startsWith("$$") ? unwrapped.text.slice(1) : unwrapped.text;
|
||||
return isRevisionName(normalized, caseInsensitive);
|
||||
}
|
||||
if (ts.isPropertyAccessExpression(unwrapped)) return isRevisionName(unwrapped.name.text, caseInsensitive);
|
||||
if (ts.isElementAccessExpression(unwrapped) && unwrapped.argumentExpression) {
|
||||
return isRevisionName(staticStringValue(unwrapped.argumentExpression), caseInsensitive);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function staticStringValue(node) {
|
||||
const expression = unwrapExpression(node);
|
||||
if (ts.isStringLiteral(expression) || ts.isNoSubstitutionTemplateLiteral(expression)) return expression.text;
|
||||
if (ts.isTemplateExpression(expression)) {
|
||||
let value = expression.head.text;
|
||||
for (const span of expression.templateSpans) {
|
||||
const part = staticStringValue(span.expression);
|
||||
if (part === undefined) return undefined;
|
||||
value += part + span.literal.text;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
if (ts.isBinaryExpression(expression) && expression.operatorToken.kind === ts.SyntaxKind.PlusToken) {
|
||||
const left = staticStringValue(expression.left);
|
||||
const right = staticStringValue(expression.right);
|
||||
return left === undefined || right === undefined ? undefined : left + right;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function propertyNameText(name, caseInsensitive = false) {
|
||||
if (!name) return undefined;
|
||||
let value;
|
||||
if (ts.isComputedPropertyName(name)) value = staticStringValue(name.expression);
|
||||
else if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNoSubstitutionTemplateLiteral(name) || ts.isNumericLiteral(name)) value = name.text;
|
||||
else value = staticStringValue(name);
|
||||
return caseInsensitive && typeof value === "string" ? value.toLowerCase() : value;
|
||||
}
|
||||
|
||||
function objectBindingHasState(pattern, caseInsensitive) {
|
||||
return pattern.elements.some((element) => {
|
||||
if (element.dotDotDotToken) return false;
|
||||
return propertyNameText(element.propertyName ?? element.name, caseInsensitive) === "state";
|
||||
});
|
||||
}
|
||||
|
||||
function objectLiteralHasState(object, caseInsensitive) {
|
||||
return object.properties.some((property) =>
|
||||
!ts.isSpreadAssignment(property) && propertyNameText(property.name, caseInsensitive) === "state");
|
||||
}
|
||||
|
||||
function scriptKindFor(path) {
|
||||
const lower = path.toLowerCase();
|
||||
if (lower.endsWith(".tsx")) return ts.ScriptKind.TSX;
|
||||
if (lower.endsWith(".jsx")) return ts.ScriptKind.JSX;
|
||||
if (/\.(?:ts|mts|cts)$/u.test(lower)) return ts.ScriptKind.TS;
|
||||
if (/\.(?:js|mjs|cjs)$/u.test(lower)) return ts.ScriptKind.JS;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function maskRange(output, source, start, end, keepEnds = false) {
|
||||
for (let cursor = start; cursor < end; cursor += 1) {
|
||||
if (source[cursor] === "\n" || source[cursor] === "\r") continue;
|
||||
if (keepEnds && (cursor === start || cursor === end - 1)) continue;
|
||||
output[cursor] = " ";
|
||||
}
|
||||
}
|
||||
|
||||
function lineEnd(source, start) {
|
||||
const end = source.indexOf("\n", start);
|
||||
return end < 0 ? source.length : end;
|
||||
}
|
||||
|
||||
function quotedEnd(source, start, delimiter, escapes = "\\") {
|
||||
for (let cursor = start + delimiter.length; cursor < source.length; cursor += 1) {
|
||||
if (escapes.includes(source[cursor])) {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
if (source.startsWith(delimiter, cursor)) return cursor + delimiter.length;
|
||||
}
|
||||
return source.length;
|
||||
}
|
||||
|
||||
function balancedEnd(source, openIndex, opener, closer, escapes = "\\`") {
|
||||
let depth = 1;
|
||||
for (let cursor = openIndex + 1; cursor < source.length; cursor += 1) {
|
||||
if (escapes.includes(source[cursor])) {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
if (source[cursor] === "'" || source[cursor] === '"' || source[cursor] === "`") {
|
||||
cursor = quotedEnd(source, cursor, source[cursor], escapes) - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[cursor] === opener) depth += 1;
|
||||
else if (source[cursor] === closer && --depth === 0) return cursor;
|
||||
}
|
||||
return source.length - 1;
|
||||
}
|
||||
|
||||
function restoreMasked(output, offset, masked) {
|
||||
for (let cursor = 0; cursor < masked.length; cursor += 1) output[offset + cursor] = masked[cursor];
|
||||
}
|
||||
|
||||
function exposeDollarSubexpressions(output, source, start, end, dialect) {
|
||||
for (let cursor = start; cursor + 1 < end; cursor += 1) {
|
||||
if (!source.startsWith("$(", cursor) || source[cursor - 1] === "`") continue;
|
||||
const close = balancedEnd(source, cursor + 1, "(", ")");
|
||||
output[cursor] = " ";
|
||||
output[cursor + 1] = "(";
|
||||
restoreMasked(output, cursor + 2, dialect === "shell" ? maskShellSource(source.slice(cursor + 2, close)) : maskPowerShellSource(source.slice(cursor + 2, close)));
|
||||
if (close < source.length) output[close] = ")";
|
||||
cursor = close;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function shellCommentStart(source, index) {
|
||||
return source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1]));
|
||||
}
|
||||
|
||||
function canonicalRevisionName(name) {
|
||||
const lower = name.toLowerCase();
|
||||
if (lower === "revision") return "revision";
|
||||
if (lower === "workspacerevision") return "workspaceRevision";
|
||||
return "selectedWorkspace";
|
||||
}
|
||||
|
||||
function normalizePowerShellVariables(source) {
|
||||
const output = source.split("");
|
||||
const patterns = [
|
||||
{ expression: /\$\{(?:[A-Za-z_][A-Za-z0-9_]*:)?(revision|workspaceRevision|selectedWorkspace)\}/giu, dollar: false },
|
||||
{ expression: /\$(?:[A-Za-z_][A-Za-z0-9_]*:)(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: false },
|
||||
{ expression: /\$(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: true },
|
||||
];
|
||||
for (const { expression, dollar } of patterns) {
|
||||
for (const match of source.matchAll(expression)) {
|
||||
const name = canonicalRevisionName(match[1]);
|
||||
const replacement = `${dollar ? "$" : ""}${name}`.padEnd(match[0].length, " ");
|
||||
for (let offset = 0; offset < match[0].length; offset += 1) output[match.index + offset] = replacement[offset];
|
||||
}
|
||||
}
|
||||
let normalized = output.join("");
|
||||
normalized = normalized.replace(/\.\s*state\b/giu, (match) => match.replace(/state/iu, "state"));
|
||||
normalized = normalized.replace(/(["'])state\1/giu, (_match, quote) => `${quote}state${quote}`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function maskShellSource(source) {
|
||||
return maskShellFamilySource(source, false);
|
||||
}
|
||||
|
||||
function maskPowerShellSource(source) {
|
||||
return normalizePowerShellVariables(maskShellFamilySource(source, true));
|
||||
}
|
||||
|
||||
function maskShellFamilySource(source, powershell) {
|
||||
const output = source.split("");
|
||||
let squareDepth = 0;
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (powershell && source.startsWith("<#", index)) {
|
||||
const close = source.indexOf("#>", index + 2);
|
||||
const end = close < 0 ? source.length : close + 2;
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (powershell ? source[index] === "#" : shellCommentStart(source, index)) {
|
||||
const end = lineEnd(source, index);
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (powershell && source[index] === "`") {
|
||||
maskRange(output, source, index, Math.min(index + 2, source.length));
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (!powershell && source[index] === "`") {
|
||||
const close = source.indexOf("`", index + 1);
|
||||
const end = close < 0 ? source.length : close + 1;
|
||||
maskRange(output, source, index, end);
|
||||
restoreMasked(output, index + 1, maskShellSource(source.slice(index + 1, close < 0 ? source.length : close)));
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
const quote = source[index];
|
||||
if (quote === "'" || quote === '"') {
|
||||
const escapes = powershell ? "`" : quote === "'" ? "" : "\\";
|
||||
const end = quotedEnd(source, index, quote, escapes);
|
||||
const preserveKey = powershell && squareDepth > 0;
|
||||
if (!preserveKey) maskRange(output, source, index, end, false);
|
||||
if (quote === '"') {
|
||||
exposeDollarSubexpressions(output, source, index + 1, end - 1, powershell ? "powershell" : "shell");
|
||||
if (!powershell) {
|
||||
for (let cursor = index + 1; cursor < end - 1; cursor += 1) {
|
||||
if (source[cursor] !== "`" || source[cursor - 1] === "\\") continue;
|
||||
const close = source.indexOf("`", cursor + 1);
|
||||
if (close < 0 || close >= end) break;
|
||||
restoreMasked(output, cursor + 1, maskShellSource(source.slice(cursor + 1, close)));
|
||||
cursor = close;
|
||||
}
|
||||
}
|
||||
}
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source.startsWith("$(", index)) output[index] = " ";
|
||||
if (source[index] === "[") squareDepth += 1;
|
||||
else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1;
|
||||
}
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function maskUnknownSource(source) {
|
||||
const output = source.split("");
|
||||
let squareDepth = 0;
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source.startsWith("/*", index)) {
|
||||
const close = source.indexOf("*/", index + 2);
|
||||
const end = close < 0 ? source.length : close + 2;
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "#" || source.startsWith("//", index)) {
|
||||
const end = lineEnd(source, index);
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
const quote = source[index];
|
||||
if (quote === "'" || quote === '"' || quote === "`") {
|
||||
const end = quotedEnd(source, index, quote, "\\");
|
||||
let after = end;
|
||||
while (/[ \t]/u.test(source[after] ?? "")) after += 1;
|
||||
if (!(squareDepth > 0 || source[after] === ":")) maskRange(output, source, index, end, true);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "[") squareDepth += 1;
|
||||
else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1;
|
||||
}
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function maskQuotedShellHeredocBodies(source, label = "<shell>") {
|
||||
const output = source.split("");
|
||||
for (const range of literalBashHeredocBodyRanges(source, label)) maskRange(output, source, range.start, range.end);
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function shellAssociativeRevisionAccess(source) {
|
||||
let quote;
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
const character = source[index];
|
||||
if (character === "\\") { index += 1; continue; }
|
||||
if (quote === "'") { if (character === "'") quote = undefined; continue; }
|
||||
if (character === "'") { quote = "'"; continue; }
|
||||
if (character === '"') { quote = quote === '"' ? undefined : '"'; continue; }
|
||||
if (character !== "$" || source[index + 1] !== "{") continue;
|
||||
const close = source.indexOf("}", index + 2);
|
||||
if (close < 0) break;
|
||||
const expansion = source.slice(index, close + 1);
|
||||
if (/^\$\{[ \t]*(?:revision|workspaceRevision|selectedWorkspace)[ \t]*\[[ \t]*(?:["']state["']|state)[ \t]*\][^}]*\}$/u.test(expansion)) return true;
|
||||
index = close;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const shellCommandPrefixes = new Set(["if", "then", "elif", "else", "while", "until", "do"]);
|
||||
const shellCommandClosers = new Set(["fi", "done", "esac"]);
|
||||
const shellControlCharacters = new Set([";", "|", "&", "(", ")", "{", "}", "`"]);
|
||||
|
||||
function shellQuotedSubstitutionEnd(source, start, depth, budget) {
|
||||
for (let index = start + 1; index < source.length; index += 1) {
|
||||
budget.characters += 1;
|
||||
if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded");
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === '"') return index + 1;
|
||||
if (source.startsWith("$(", index) || source.startsWith("<(", index) || source.startsWith(">(", index)) {
|
||||
index = shellParenthesizedEnd(source, index + 1, depth + 1, budget) - 1;
|
||||
} else if (source[index] === "`") {
|
||||
const end = quotedEnd(source, index, "`", "\\");
|
||||
if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick");
|
||||
index = end - 1;
|
||||
}
|
||||
}
|
||||
throw new Error("revision-state shell substitution has an unclosed quote");
|
||||
}
|
||||
|
||||
function shellParenthesizedEnd(source, openIndex, depth, budget) {
|
||||
if (depth > 64) throw new Error("revision-state shell substitution nesting limit exceeded");
|
||||
for (let index = openIndex + 1; index < source.length; index += 1) {
|
||||
budget.characters += 1;
|
||||
if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded");
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === "'") {
|
||||
const end = quotedEnd(source, index, "'", "");
|
||||
if (end - 1 <= index || source[end - 1] !== "'") throw new Error("revision-state shell substitution has an unclosed quote");
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === '"') { index = shellQuotedSubstitutionEnd(source, index, depth, budget) - 1; continue; }
|
||||
if (source[index] === "`") {
|
||||
const end = quotedEnd(source, index, "`", "\\");
|
||||
if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick");
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "#" && (index === openIndex + 1 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) {
|
||||
index = lineEnd(source, index);
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "(") { index = shellParenthesizedEnd(source, index, depth + 1, budget) - 1; continue; }
|
||||
if (source[index] === ")") return index + 1;
|
||||
}
|
||||
throw new Error("revision-state shell process substitution is unbalanced");
|
||||
}
|
||||
|
||||
function shellProcessSubstitutionEnd(source, start) {
|
||||
if (!(source.startsWith("<(", start) || source.startsWith(">(", start))) return undefined;
|
||||
return shellParenthesizedEnd(source, start + 1, 1, { characters: 0 });
|
||||
}
|
||||
|
||||
function shellRedirectionAt(source, start) {
|
||||
const match = source.slice(start).match(/^(?:&>>|&>|(?:[0-9]+|\{[A-Za-z_][A-Za-z0-9_]*\})?(?:<<<|<<-|<<|>>|<>|>\||<&|>&|<|>))/u);
|
||||
if (!match) return undefined;
|
||||
let end = start + match[0].length;
|
||||
while (end < source.length && !/\s/u.test(source[end]) && !shellControlCharacters.has(source[end]) &&
|
||||
source[end] !== "<" && source[end] !== ">" && source[end] !== "'" && source[end] !== '"') end += 1;
|
||||
return { value: source.slice(start, end), end, needsOperand: end === start + match[0].length };
|
||||
}
|
||||
|
||||
function shellLexTokens(source) {
|
||||
const tokens = [];
|
||||
const push = (value, start, end, type = "word") => {
|
||||
tokens.push({ value, start, end, type });
|
||||
if (tokens.length > 50_000) throw new Error("revision-state shell token limit exceeded");
|
||||
};
|
||||
for (let index = 0; index < source.length;) {
|
||||
if (source[index] === "\n" || source[index] === "\r") { push(source[index], index, index + 1, "control"); index += 1; continue; }
|
||||
if (/\s/u.test(source[index])) { index += 1; continue; }
|
||||
if (source[index] === "#") { index = lineEnd(source, index); continue; }
|
||||
const processEnd = shellProcessSubstitutionEnd(source, index);
|
||||
if (processEnd !== undefined) {
|
||||
push(source.slice(index, processEnd), index, processEnd);
|
||||
index = processEnd;
|
||||
continue;
|
||||
}
|
||||
const redirection = shellRedirectionAt(source, index);
|
||||
if (redirection) {
|
||||
push(redirection.value, index, redirection.end, "redirection");
|
||||
tokens.at(-1).needsOperand = redirection.needsOperand;
|
||||
index = redirection.end;
|
||||
continue;
|
||||
}
|
||||
if (shellControlCharacters.has(source[index]) || source[index] === "!" && (index === 0 || /\s/u.test(source[index - 1]))) {
|
||||
const start = index;
|
||||
let value = source[index++];
|
||||
if ((value === ";" || value === "|" || value === "&") && source[index] === value) value += source[index++];
|
||||
push(value, start, index, "control");
|
||||
continue;
|
||||
}
|
||||
const start = index;
|
||||
let value = "";
|
||||
while (index < source.length && !/\s/u.test(source[index]) && !shellControlCharacters.has(source[index]) && source[index] !== "<" && source[index] !== ">") {
|
||||
const quote = source[index];
|
||||
if (quote === "'" || quote === '"') {
|
||||
const end = quotedEnd(source, index, quote, "\\");
|
||||
value += source.slice(index + 1, end - 1);
|
||||
index = end;
|
||||
} else if (source[index] === "\\" && index + 1 < source.length) {
|
||||
value += source[index + 1];
|
||||
index += 2;
|
||||
} else {
|
||||
value += source[index++];
|
||||
}
|
||||
}
|
||||
push(value, start, index);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function shellCommandWords(source) {
|
||||
const commands = [];
|
||||
let words = [];
|
||||
const finish = () => { if (words.length > 0) commands.push(words); words = []; };
|
||||
for (const token of shellLexTokens(source)) {
|
||||
if (token.type === "control") {
|
||||
finish();
|
||||
continue;
|
||||
}
|
||||
if (token.type === "word" && words.length === 0 && shellCommandPrefixes.has(token.value)) continue;
|
||||
if (token.type === "word" && words.length === 0 && shellCommandClosers.has(token.value)) continue;
|
||||
words.push(token);
|
||||
}
|
||||
finish();
|
||||
return commands;
|
||||
}
|
||||
|
||||
function shellExecutable(word) {
|
||||
return word?.split("/").pop();
|
||||
}
|
||||
|
||||
const shellWrapperSpecs = new Map([
|
||||
["command", { kind: "options", operandOptions: new Set() }],
|
||||
["env", { kind: "env", operandOptions: new Set(["-u", "--unset", "-C", "--chdir"]) }],
|
||||
["sudo", { kind: "options", operandOptions: new Set(["-u", "--user", "-g", "--group", "-h", "--host", "-p", "--prompt", "-C", "--close-from", "-D", "--chdir"]) }],
|
||||
["nice", { kind: "options", operandOptions: new Set(["-n", "--adjustment"]) }],
|
||||
["time", { kind: "options", operandOptions: new Set(["-o", "--output", "-f", "--format"]) }],
|
||||
["xargs", { kind: "options", operandOptions: new Set(["-I", "--replace", "-n", "--max-args", "-L", "--max-lines", "-P", "--max-procs", "-s", "--max-chars", "-d", "--delimiter"]) }],
|
||||
["timeout", { kind: "timeout", operandOptions: new Set(["-k", "--kill-after", "-s", "--signal"]) }],
|
||||
["stdbuf", { kind: "stdbuf", operandOptions: new Set(["-i", "--input", "-o", "--output", "-e", "--error"]) }],
|
||||
["nohup", { kind: "options", operandOptions: new Set() }],
|
||||
["exec", { kind: "options", operandOptions: new Set(["-a"]) }],
|
||||
["coproc", { kind: "coproc", operandOptions: new Set() }],
|
||||
]);
|
||||
|
||||
function skipShellMetadata(words, start) {
|
||||
let index = start;
|
||||
while (index < words.length) {
|
||||
const token = words[index];
|
||||
if (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(token.value)) { index += 1; continue; }
|
||||
if (token.type === "redirection") { index += token.needsOperand ? 2 : 1; continue; }
|
||||
break;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
function skipWrapperOptions(words, start, spec) {
|
||||
let index = start;
|
||||
while (index < words.length) {
|
||||
const word = words[index].value;
|
||||
if (word === "--") return index + 1;
|
||||
if (spec.operandOptions.has(word)) { index += 2; continue; }
|
||||
if (spec.kind === "stdbuf" && /^-(?:i|o|e).+/u.test(word)) { index += 1; continue; }
|
||||
if (word.startsWith("-")) { index += 1; continue; }
|
||||
break;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
function shellJqArguments(words) {
|
||||
let index = skipShellMetadata(words, 0);
|
||||
let wrappers = 0;
|
||||
while (index < words.length) {
|
||||
const spec = shellWrapperSpecs.get(shellExecutable(words[index]?.value));
|
||||
if (!spec) break;
|
||||
if (wrappers >= 16) throw new Error("revision-state shell wrapper nesting exceeds policy limit");
|
||||
wrappers += 1;
|
||||
index = skipWrapperOptions(words, index + 1, spec);
|
||||
if (spec.kind === "env") {
|
||||
while (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(words[index]?.value ?? "")) index += 1;
|
||||
} else if (spec.kind === "timeout") {
|
||||
if (index >= words.length) return undefined;
|
||||
index += 1;
|
||||
} else if (spec.kind === "coproc") {
|
||||
index = skipShellMetadata(words, index);
|
||||
const current = shellExecutable(words[index]?.value);
|
||||
if (current !== "jq" && !shellWrapperSpecs.has(current) && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(words[index]?.value ?? "")) {
|
||||
const afterName = skipShellMetadata(words, index + 1);
|
||||
const command = shellExecutable(words[afterName]?.value);
|
||||
if (command === "jq" || shellWrapperSpecs.has(command)) index = afterName;
|
||||
}
|
||||
}
|
||||
index = skipShellMetadata(words, index);
|
||||
}
|
||||
return shellExecutable(words[index]?.value) === "jq" ? words.slice(index + 1) : undefined;
|
||||
}
|
||||
|
||||
const jqOptionOperands = new Map([
|
||||
["--arg", 2], ["--argjson", 2], ["--slurpfile", 2], ["--rawfile", 2], ["--argfile", 2],
|
||||
["-L", 1], ["--library-path", 1], ["--indent", 1],
|
||||
["-f", 1], ["--from-file", 1],
|
||||
]);
|
||||
const jqFileFilterOptions = new Set(["-f", "--from-file"]);
|
||||
|
||||
function withoutShellRedirections(arguments_) {
|
||||
const semantic = [];
|
||||
for (let index = 0; index < arguments_.length; index += 1) {
|
||||
const token = arguments_[index];
|
||||
if (token.type === "redirection") { if (token.needsOperand) index += 1; continue; }
|
||||
semantic.push(token);
|
||||
}
|
||||
return semantic;
|
||||
}
|
||||
|
||||
function jqInvocation(arguments_) {
|
||||
const semantic = withoutShellRedirections(arguments_);
|
||||
let fromFile = false;
|
||||
for (let index = 0; index < semantic.length; index += 1) {
|
||||
const argument = semantic[index].value;
|
||||
if (argument === "--") return { filter: fromFile ? undefined : semantic[index + 1], arguments_ };
|
||||
const operands = jqOptionOperands.get(argument);
|
||||
if (operands !== undefined) {
|
||||
if (jqFileFilterOptions.has(argument)) fromFile = true;
|
||||
index += operands;
|
||||
continue;
|
||||
}
|
||||
if (argument.startsWith("-")) continue;
|
||||
return { filter: fromFile ? undefined : semantic[index], arguments_ };
|
||||
}
|
||||
return { filter: undefined, arguments_ };
|
||||
}
|
||||
|
||||
function maskShellJqLiteralArguments(source) {
|
||||
const output = source.split("");
|
||||
for (const words of shellCommandWords(source)) {
|
||||
const arguments_ = shellJqArguments(words);
|
||||
if (!arguments_) continue;
|
||||
const invocation = jqInvocation(arguments_);
|
||||
for (const argument of invocation.arguments_) {
|
||||
if (argument === invocation.filter) continue;
|
||||
const raw = source.slice(argument.start, argument.end);
|
||||
if (!raw.includes("$") && !raw.includes("`")) maskRange(output, source, argument.start, argument.end);
|
||||
}
|
||||
}
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function jqStringEnd(source, start) {
|
||||
for (let index = start + 1; index < source.length; index += 1) {
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === '"') return index;
|
||||
}
|
||||
return source.length;
|
||||
}
|
||||
|
||||
function jqInterpolationEnd(source, start) {
|
||||
let depth = 1;
|
||||
for (let index = start; index < source.length; index += 1) {
|
||||
if (source[index] === '"') { index = jqStringEnd(source, index); continue; }
|
||||
if (source[index] === "(") depth += 1;
|
||||
else if (source[index] === ")" && --depth === 0) return index;
|
||||
}
|
||||
return source.length;
|
||||
}
|
||||
|
||||
function jqTokens(source, budget = { tokens: 0, depth: 0 }) {
|
||||
if (budget.depth >= 64) throw new Error("jq filter exceeds policy nesting limit");
|
||||
budget.depth += 1;
|
||||
const tokens = [];
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
budget.tokens += 1;
|
||||
if (budget.tokens >= 10_000) throw new Error("jq filter exceeds policy token limit");
|
||||
if (/\s/u.test(source[index])) continue;
|
||||
if (source[index] === "#") { index = lineEnd(source, index); continue; }
|
||||
if (source[index] === '"') {
|
||||
const end = jqStringEnd(source, index);
|
||||
const raw = source.slice(index, Math.min(end + 1, source.length));
|
||||
let value;
|
||||
if (!raw.includes("\\(")) {
|
||||
try { value = JSON.parse(raw); } catch { value = undefined; }
|
||||
}
|
||||
tokens.push({ type: "string", value });
|
||||
for (let cursor = index + 1; cursor < end; cursor += 1) {
|
||||
if (source[cursor] === "\\" && source[cursor + 1] === "(") {
|
||||
const close = jqInterpolationEnd(source, cursor + 2);
|
||||
tokens.push(...jqTokens(source.slice(cursor + 2, close), budget));
|
||||
cursor = close;
|
||||
} else if (source[cursor] === "\\") cursor += 1;
|
||||
}
|
||||
index = end;
|
||||
continue;
|
||||
}
|
||||
const variable = source.slice(index).match(/^\$([A-Za-z_][A-Za-z0-9_]*)/u);
|
||||
if (variable) { tokens.push({ type: "variable", value: variable[1] }); index += variable[0].length - 1; continue; }
|
||||
const identifier = source.slice(index).match(/^[A-Za-z_][A-Za-z0-9_]*/u);
|
||||
if (identifier) { tokens.push({ type: "identifier", value: identifier[0] }); index += identifier[0].length - 1; continue; }
|
||||
const punctuation = { ".": "dot", "[": "open", "]": "close" }[source[index]];
|
||||
tokens.push({ type: punctuation ?? "other", value: source[index] });
|
||||
}
|
||||
budget.depth -= 1;
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function jqStaticString(tokens, cursor, depth = 0) {
|
||||
if (depth >= 64) throw new Error("revision-state jq static-key nesting exceeds policy limit");
|
||||
let index = cursor;
|
||||
let value;
|
||||
if (tokens[index]?.type === "string" && typeof tokens[index].value === "string") {
|
||||
value = tokens[index].value;
|
||||
index += 1;
|
||||
} else if (tokens[index]?.type === "other" && tokens[index].value === "(") {
|
||||
const nested = jqStaticString(tokens, index + 1, depth + 1);
|
||||
if (!nested || tokens[nested.next]?.type !== "other" || tokens[nested.next].value !== ")") return undefined;
|
||||
value = nested.value;
|
||||
index = nested.next + 1;
|
||||
} else return undefined;
|
||||
while (tokens[index]?.type === "other" && tokens[index].value === "+") {
|
||||
const right = jqStaticString(tokens, index + 1, depth + 1);
|
||||
if (!right) return undefined;
|
||||
value += right.value;
|
||||
index = right.next;
|
||||
}
|
||||
return { value, next: index };
|
||||
}
|
||||
|
||||
function jqBracketSegment(tokens, cursor) {
|
||||
if (tokens[cursor]?.type !== "open") return undefined;
|
||||
const expression = jqStaticString(tokens, cursor + 1);
|
||||
return expression && tokens[expression.next]?.type === "close" ?
|
||||
{ value: expression.value, next: expression.next + 1 } : undefined;
|
||||
}
|
||||
|
||||
function jqPathSegment(tokens, cursor, allowBareBracket = true) {
|
||||
if (tokens[cursor]?.type === "variable") return { value: tokens[cursor].value, next: cursor + 1 };
|
||||
let index = cursor;
|
||||
if (tokens[index]?.type === "dot") {
|
||||
index += 1;
|
||||
if (tokens[index]?.type === "identifier" || tokens[index]?.type === "string") return { value: tokens[index].value, next: index + 1 };
|
||||
}
|
||||
return allowBareBracket ? jqBracketSegment(tokens, index) : undefined;
|
||||
}
|
||||
|
||||
function jqIdentityPipelineEnd(tokens, cursor) {
|
||||
let index = cursor;
|
||||
while (tokens[index]?.type === "other" && tokens[index].value === "(") index += 1;
|
||||
if (tokens[index]?.type !== "dot") return undefined;
|
||||
index += 1;
|
||||
while (tokens[index]?.type === "other" && tokens[index].value === ")") index += 1;
|
||||
return tokens[index]?.type === "other" && tokens[index].value === "|" ? index + 1 : undefined;
|
||||
}
|
||||
|
||||
function jqTargetGrammarSupported(tokens) {
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
const token = tokens[index];
|
||||
if (token.type === "identifier" && tokens[index - 1]?.type !== "dot") return false;
|
||||
if (token.type === "open" && !jqBracketSegment(tokens, index)) return false;
|
||||
if (token.type !== "other") continue;
|
||||
if (["?", "(", ")", "|"].includes(token.value)) continue;
|
||||
if (token.value === "+" && (tokens[index - 1]?.type === "string" || tokens[index - 1]?.value === ")") &&
|
||||
(tokens[index + 1]?.type === "string" || tokens[index + 1]?.value === "(")) continue;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function jqContainsActiveTarget(tokens) {
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
if (tokens[index].type === "variable" && revisionIdentifiers.has(tokens[index].value)) return true;
|
||||
if (tokens[index].type === "dot" && (tokens[index + 1]?.type === "identifier" || tokens[index + 1]?.type === "string") &&
|
||||
revisionIdentifiers.has(tokens[index + 1].value)) return true;
|
||||
if (tokens[index].type === "open" && (tokens[index - 1]?.type === "dot" || tokens[index - 1]?.type === "close" || tokens[index - 1]?.type === "identifier")) {
|
||||
const key = jqStaticString(tokens, index + 1);
|
||||
if (key && revisionIdentifiers.has(key.value)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function jqRevisionAnalysis(filter) {
|
||||
const tokens = jqTokens(filter);
|
||||
let activeTarget = jqContainsActiveTarget(tokens);
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
if (tokens[index].type !== "dot" && tokens[index].type !== "variable") continue;
|
||||
const segments = [];
|
||||
let cursor = index;
|
||||
let pipelineBoundary = false;
|
||||
while (cursor < tokens.length) {
|
||||
if (pipelineBoundary && (tokens[cursor]?.type === "open" || tokens[cursor]?.type === "string")) {
|
||||
segments.length = 0;
|
||||
break;
|
||||
}
|
||||
if (pipelineBoundary && tokens[cursor]?.type === "variable") segments.length = 0;
|
||||
const segment = jqPathSegment(tokens, cursor, !pipelineBoundary);
|
||||
if (!segment) break;
|
||||
pipelineBoundary = false;
|
||||
segments.push(segment.value);
|
||||
cursor = segment.next;
|
||||
while (tokens[cursor]?.type === "other" && tokens[cursor].value === "?") cursor += 1;
|
||||
while (tokens[cursor]?.type === "other" && tokens[cursor].value === ")") cursor += 1;
|
||||
if (tokens[cursor]?.type === "other" && tokens[cursor].value === "|") {
|
||||
cursor += 1;
|
||||
while (tokens[cursor]?.type === "other" && tokens[cursor].value === "(") cursor += 1;
|
||||
let identityEnd;
|
||||
while ((identityEnd = jqIdentityPipelineEnd(tokens, cursor)) !== undefined) cursor = identityEnd;
|
||||
pipelineBoundary = true;
|
||||
}
|
||||
}
|
||||
if (segments.some((segment) => revisionIdentifiers.has(segment))) activeTarget = true;
|
||||
for (let position = 0; position + 1 < segments.length; position += 1) {
|
||||
if (revisionIdentifiers.has(segments[position]) && segments[position + 1] === "state") return "violation";
|
||||
}
|
||||
}
|
||||
if (!activeTarget) return "safe";
|
||||
return jqTargetGrammarSupported(tokens) ? "safe" : "unsupported";
|
||||
}
|
||||
|
||||
function shellExecutableSubstitutionBodies(source, arithmeticContext = false) {
|
||||
const bodies = [];
|
||||
const addParenthesized = (start, kind) => {
|
||||
const end = shellParenthesizedEnd(source, start + 1, 1, { characters: 0 });
|
||||
bodies.push({ kind, start: start + 2, end: end - 1, source: source.slice(start + 2, end - 1) });
|
||||
return end;
|
||||
};
|
||||
const addBacktick = (start) => {
|
||||
const end = quotedEnd(source, start, "`", "\\");
|
||||
if (end - 1 <= start || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick");
|
||||
bodies.push({ kind: "backtick", start: start + 1, end: end - 1, source: source.slice(start + 1, end - 1) });
|
||||
return end;
|
||||
};
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) { index = lineEnd(source, index); continue; }
|
||||
if (source[index] === "'") {
|
||||
const end = quotedEnd(source, index, "'", "");
|
||||
if (end - 1 <= index || source[end - 1] !== "'") throw new Error(`revision-state shell policy found an unclosed quote at offset ${index}`);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === '"') {
|
||||
for (let cursor = index + 1; cursor < source.length; cursor += 1) {
|
||||
if (source[cursor] === "\\") { cursor += 1; continue; }
|
||||
if (source[cursor] === '"') { index = cursor; break; }
|
||||
if (source.startsWith("$(", cursor)) {
|
||||
const end = addParenthesized(cursor, source.startsWith("$((", cursor) ? "arithmetic" : "command");
|
||||
cursor = end - 1;
|
||||
} else if (source[cursor] === "`") {
|
||||
cursor = addBacktick(cursor) - 1;
|
||||
}
|
||||
if (cursor + 1 >= source.length) throw new Error(`revision-state shell policy found an unclosed double quote at offset ${index}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!arithmeticContext && (source.startsWith("<(", index) || source.startsWith(">(", index))) {
|
||||
index = addParenthesized(index, "process") - 1;
|
||||
continue;
|
||||
}
|
||||
if (source.startsWith("$(", index)) {
|
||||
const arithmetic = source.startsWith("$((", index);
|
||||
index = addParenthesized(index, arithmetic ? "arithmetic" : "command") - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "`") index = addBacktick(index) - 1;
|
||||
}
|
||||
return bodies;
|
||||
}
|
||||
|
||||
function removeBacktickBodyEscapes(source) {
|
||||
let result = "";
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source[index] === "\\" && index + 1 < source.length && ["$", "`", "\\", "\n"].includes(source[index + 1])) {
|
||||
if (source[index + 1] !== "\n") result += source[index + 1];
|
||||
index += 1;
|
||||
} else {
|
||||
result += source[index];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function shellJqRevisionAccess(source, budget = { characters: 0 }, depth = 0, arithmeticContext = false) {
|
||||
if (depth > 32) throw new Error("revision-state executable shell substitution nesting limit exceeded");
|
||||
budget.characters += source.length;
|
||||
if (budget.characters > 500_000) throw new Error("revision-state executable shell substitution size limit exceeded");
|
||||
if (!arithmeticContext) {
|
||||
for (const words of shellCommandWords(source)) {
|
||||
const arguments_ = shellJqArguments(words);
|
||||
const filter = arguments_ && jqInvocation(arguments_).filter;
|
||||
if (filter) {
|
||||
const analysis = jqRevisionAnalysis(filter.value);
|
||||
if (analysis === "violation") return true;
|
||||
if (analysis === "unsupported") throw new Error("revision-state jq target grammar is unsupported");
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const body of shellExecutableSubstitutionBodies(source, arithmeticContext)) {
|
||||
const nestedSource = body.kind === "backtick" ? removeBacktickBodyEscapes(body.source) : body.source;
|
||||
if (shellJqRevisionAccess(nestedSource, budget, depth + 1, body.kind === "arithmetic")) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function nonJsAnalysisSource(source, label) {
|
||||
const lower = label.toLowerCase();
|
||||
if (lower.endsWith(".sh")) return maskShellSource(maskShellJqLiteralArguments(maskQuotedShellHeredocBodies(source, label)));
|
||||
if (lower.endsWith(".ps1")) return maskPowerShellSource(source);
|
||||
return maskUnknownSource(source);
|
||||
}
|
||||
|
||||
function revisionStateAstNodes(source, label) {
|
||||
const knownKind = scriptKindFor(label);
|
||||
const caseInsensitive = label.toLowerCase().endsWith(".ps1");
|
||||
const analyzed = knownKind === undefined ? nonJsAnalysisSource(source, label) : source;
|
||||
const file = ts.createSourceFile(label, analyzed, ts.ScriptTarget.Latest, true, knownKind ?? ts.ScriptKind.TS);
|
||||
const matches = [];
|
||||
function visit(node) {
|
||||
if (ts.isPropertyAccessExpression(node) && node.name.text === "state" && isRevisionExpression(node.expression, caseInsensitive)) {
|
||||
matches.push(node);
|
||||
} else if (ts.isElementAccessExpression(node) && isRevisionExpression(node.expression, caseInsensitive) &&
|
||||
node.argumentExpression && propertyNameText(node.argumentExpression, caseInsensitive) === "state") {
|
||||
matches.push(node);
|
||||
} else if ((ts.isVariableDeclaration(node) || ts.isParameter(node)) && node.initializer &&
|
||||
isRevisionExpression(node.initializer, caseInsensitive) && ts.isObjectBindingPattern(node.name) &&
|
||||
objectBindingHasState(node.name, caseInsensitive)) {
|
||||
matches.push(node);
|
||||
} else if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&
|
||||
isRevisionExpression(node.right, caseInsensitive)) {
|
||||
const assignmentTarget = unwrapExpression(node.left);
|
||||
if (ts.isObjectLiteralExpression(assignmentTarget) && objectLiteralHasState(assignmentTarget, caseInsensitive)) matches.push(node);
|
||||
} else if (ts.isPropertyAssignment(node) && propertyNameText(node.name, caseInsensitive) === "revision" &&
|
||||
ts.isObjectLiteralExpression(node.initializer) && objectLiteralHasState(node.initializer, caseInsensitive)) {
|
||||
matches.push(node);
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
visit(file);
|
||||
return matches;
|
||||
}
|
||||
|
||||
|
||||
function yamlScalarRevisionAccess(value) {
|
||||
return /(?:^|[\s;=,(])(?:revision|workspaceRevision|selectedWorkspace)\s*(?:\.\s*state|\[\s*["']?state["']?\s*\])(?:$|[\s;,)])/u.test(value);
|
||||
}
|
||||
|
||||
function validateYamlRevisionState(source, label) {
|
||||
const documents = parseAllDocuments(source, { uniqueKeys: true, merge: true });
|
||||
for (const document of documents) {
|
||||
if (document.errors.length > 0) throw new Error(`${label}: revision-state policy cannot parse YAML`);
|
||||
const walkAst = (node) => {
|
||||
if (isScalar(node)) {
|
||||
if (node.type === "PLAIN" && typeof node.value === "string" && yamlScalarRevisionAccess(node.value)) throw new Error(`${label}: forbidden revision-state access`);
|
||||
return;
|
||||
}
|
||||
if (isSeq(node)) { for (const item of node.items) walkAst(item); return; }
|
||||
if (isMap(node)) { for (const pair of node.items) walkAst(pair.value); }
|
||||
};
|
||||
walkAst(document.contents);
|
||||
let resolved;
|
||||
try { resolved = document.toJS({ mapAsMap: true, maxAliasCount: 50 }); }
|
||||
catch { throw new Error(`${label}: revision-state YAML alias resolution failed`); }
|
||||
const seen = new WeakSet();
|
||||
const walkResolved = (value) => {
|
||||
if (!value || typeof value !== "object" || seen.has(value)) return;
|
||||
seen.add(value);
|
||||
if (value instanceof Map) {
|
||||
for (const [key, child] of value) {
|
||||
if (revisionIdentifiers.has(String(key)) && child instanceof Map && child.has("state")) throw new Error(`${label}: forbidden revision-state access`);
|
||||
walkResolved(child);
|
||||
}
|
||||
} else if (Array.isArray(value)) { for (const child of value) walkResolved(child); }
|
||||
};
|
||||
walkResolved(resolved);
|
||||
}
|
||||
}
|
||||
|
||||
function validateRevisionState(source, label) {
|
||||
const lower = label.toLowerCase();
|
||||
if (/\.(?:yaml|yml)(?:\.example)?$/u.test(lower)) {
|
||||
validateYamlRevisionState(source, label);
|
||||
return;
|
||||
}
|
||||
if (lower.endsWith(".sh")) {
|
||||
const active = maskQuotedShellHeredocBodies(source, label);
|
||||
try {
|
||||
if (shellJqRevisionAccess(active) || shellAssociativeRevisionAccess(active)) throw new Error("forbidden revision-state access");
|
||||
} catch (error) {
|
||||
throw new Error(`${label}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
if (lower.endsWith(".py") || lower.endsWith(".pyw")) throw new Error(`${label}: revision-state Python input was not batched`);
|
||||
const matches = revisionStateAstNodes(source, label);
|
||||
if (matches.length === 0) return;
|
||||
const historical = 'revision.state !== "operational"';
|
||||
const historicalCount = source.split(historical).length - 1;
|
||||
const match = matches[0];
|
||||
if (label === "backend/src/workspaces/registry.ts" && matches.length === 1 &&
|
||||
match.getText() === "revision.state" && match.parent?.getText() === historical &&
|
||||
historicalCount === 1) return;
|
||||
throw new Error(`${label}: forbidden revision-state access`);
|
||||
}
|
||||
|
||||
|
||||
const pythonHelper = fileURLToPath(new URL("./revision_state_policy.py", import.meta.url));
|
||||
|
||||
function validatePythonRevisionStates(records) {
|
||||
if (!Array.isArray(records) || records.length === 0) return;
|
||||
let stdout;
|
||||
try {
|
||||
stdout = execFileSync("python3", ["-I", "-B", pythonHelper], {
|
||||
input: JSON.stringify(records), encoding: "utf8", timeout: 5_000, maxBuffer: 4 * 1024 * 1024,
|
||||
env: {
|
||||
PATH: process.env.PATH ?? "/usr/bin:/bin",
|
||||
LANG: "C.UTF-8",
|
||||
LC_ALL: "C.UTF-8",
|
||||
PYTHONDONTWRITEBYTECODE: "1",
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
} catch (error) {
|
||||
const detail = error?.stderr?.toString().trim();
|
||||
throw new Error(`revision-state helper failed${detail ? `: ${detail}` : ""}`);
|
||||
}
|
||||
let result;
|
||||
try { result = JSON.parse(stdout); }
|
||||
catch { throw new Error("revision-state helper failed: invalid JSON output"); }
|
||||
if (!result || !Array.isArray(result.violations) || result.violations.some((label) => typeof label !== "string")) throw new Error("revision-state helper failed: invalid result shape");
|
||||
if (result.violations.length > 0) throw new Error(`${result.violations[0]}: forbidden revision-state access`);
|
||||
}
|
||||
|
||||
export { validatePythonRevisionStates, validateRevisionState };
|
||||
@@ -1,273 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs";
|
||||
|
||||
function rejects(source, path) {
|
||||
assert.throws(() => validateRevisionState(source, path), /revision-state/, source);
|
||||
}
|
||||
function passes(source, path) {
|
||||
assert.doesNotThrow(() => validateRevisionState(source, path));
|
||||
}
|
||||
|
||||
test("PowerShell scoped and braced revision variables remain executable", () => {
|
||||
rejects('${revision}.state', "scripts/direct.ps1");
|
||||
rejects('${workspaceRevision}["state"]', "scripts/bracket.ps1");
|
||||
rejects('Write-Output "$(${selectedWorkspace}.state)"', "scripts/subexpression.ps1");
|
||||
rejects('${script:revision}.state', "scripts/scoped.ps1");
|
||||
rejects('${global:workspaceRevision}["state"]', "scripts/global.ps1");
|
||||
for (const source of [
|
||||
'$REVISION.STATE',
|
||||
'${Revision}.state',
|
||||
'$WORKSPACEREVISION["STATE"]',
|
||||
'${GLOBAL:SELECTEDWORKSPACE}.State',
|
||||
'$REVISION["ST" + "ATE"]',
|
||||
'${Revision}[("sT" + "AtE")]',
|
||||
'$record.REVISION.STATE',
|
||||
'$record.WORKSPACEREVISION["STATE"]',
|
||||
'$record["REVISION"].STATE',
|
||||
]) rejects(source, "scripts/case.ps1");
|
||||
passes('REVISION.STATE; revision.STATE; revision["ST" + "ATE"]; record.REVISION.STATE; record["REVISION"].state', "backend/src/case-sensitive.ts");
|
||||
});
|
||||
|
||||
test("Bash jq command forms and associative revision parameters are active", () => {
|
||||
for (const source of [
|
||||
"value=$(jq -r '.revision.state' snapshot.json)",
|
||||
"value=$(command jq -r '.workspaceRevision.state' snapshot.json)",
|
||||
"/usr/bin/jq --arg x y '.selectedWorkspace.state' snapshot.json",
|
||||
"env -i MODE=x jq -- '.revision.state' snapshot.json",
|
||||
"env -u MODE /opt/tools/jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"echo safe\nvalue=`jq -r '.selectedWorkspace.state' snapshot.json`",
|
||||
"sudo -u nobody /usr/bin/jq -r '.revision.state' snapshot.json",
|
||||
"nice -n 5 jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"time jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
"printf x | xargs -n 1 jq -r '.revision.state'",
|
||||
"timeout -k 2 5 jq -r '.revision.state' snapshot.json",
|
||||
`stdbuf -o L jq -r '.["workspaceRevision"].state' snapshot.json`,
|
||||
`stdbuf -oL jq -r '.["selectedWorkspace"]["state"]' snapshot.json`,
|
||||
`nohup jq -r '.revision["state"]' snapshot.json`,
|
||||
"< snapshot.json jq -r '.workspaceRevision.state'",
|
||||
"sudo MODE=x jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
String.raw`jq -r '"x \(.revision.state)"' snapshot.json`,
|
||||
"jq < snapshot.json -r '.revision.state'",
|
||||
"jq -r < snapshot.json '.workspaceRevision.state'",
|
||||
"jq --arg note safe < snapshot.json '.selectedWorkspace.state'",
|
||||
"jq -r '.revision?.state' snapshot.json",
|
||||
`jq -r '.["workspaceRevision"]?["state"]' snapshot.json`,
|
||||
`jq -r '.["revision"]?.["state"]' snapshot.json`,
|
||||
`jq -r '."revision".state' snapshot.json`,
|
||||
`jq -r '."workspaceRevision"."state"' snapshot.json`,
|
||||
"jq -r '$revision.state' snapshot.json",
|
||||
"jq -r '($selectedWorkspace).state' snapshot.json",
|
||||
`${"env ".repeat(17)}jq -r '.revision.state' snapshot.json`,
|
||||
"jq<input.json -r '.revision.state'",
|
||||
"jq 2>/dev/null -r '.workspaceRevision.state' snapshot.json",
|
||||
"{ jq -r '.selectedWorkspace.state' snapshot.json; }",
|
||||
"! jq -r '.revision.state' snapshot.json",
|
||||
"if jq -r '.workspaceRevision.state' snapshot.json; then :; fi",
|
||||
"if false; then :; elif jq -r '.selectedWorkspace.state' snapshot.json; then :; fi",
|
||||
"while false; do jq -r '.revision.state' snapshot.json; done",
|
||||
"until false; do jq -r '.workspaceRevision.state' snapshot.json; done",
|
||||
"jq -r '.revision | .state' snapshot.json",
|
||||
"jq -r '(.workspaceRevision | .state)' snapshot.json",
|
||||
`jq -r '.["revi" + "sion"].state' snapshot.json`,
|
||||
`jq -r '.["workspace" + "Revision"]["st" + "ate"]' snapshot.json`,
|
||||
"jq 2>&1 -r '.revision.state' snapshot.json",
|
||||
"jq 2>&- -r '.workspaceRevision.state' snapshot.json",
|
||||
"jq 0<&3 -r '.selectedWorkspace.state' snapshot.json",
|
||||
"jq &>/dev/null -r '.revision.state' snapshot.json",
|
||||
"jq &>>log -r '.workspaceRevision.state' snapshot.json",
|
||||
"jq >|output -r '.selectedWorkspace.state' snapshot.json",
|
||||
"jq {fd}>output -r '.revision.state' snapshot.json",
|
||||
"exec jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"coproc jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
"coproc worker jq -r '.revision.state' snapshot.json",
|
||||
"coproc worker >out jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"coproc worker 2>/dev/null jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
"coproc worker VAR=x jq -r '.revision.state' snapshot.json",
|
||||
`jq -r '.["revi" + ("sion")].state' snapshot.json`,
|
||||
"jq -r '.revision | . | .state' snapshot.json",
|
||||
"jq -r '(.workspaceRevision | (.) | .state)' snapshot.json",
|
||||
"jq -r '.revision | select(.) | .state' snapshot.json",
|
||||
"jq -r '.workspaceRevision | {value:.state}' snapshot.json",
|
||||
"jq -r '.selectedWorkspace | [.state]' snapshot.json",
|
||||
"jq -r '.revision + .state' snapshot.json",
|
||||
"jq < <(cat snapshot.json) -r '.revision.state'",
|
||||
"jq < <(cat <(printf snapshot.json)) -r '.workspaceRevision.state'",
|
||||
"jq > >(cat >/dev/null) -r '.selectedWorkspace.state' snapshot.json",
|
||||
`jq < <(printf '%s\n' "$((1 + (2)))") -r '.revision.state'`,
|
||||
"jq < <(cat snapshot.json -r '.revision.state'",
|
||||
`${"<(".repeat(65)}echo snapshot${")".repeat(65)} jq -r '.workspaceRevision.state'`,
|
||||
"cat <(jq -r '.revision.state' snapshot.json)",
|
||||
"cat snapshot.json > >(jq -r '.workspaceRevision.state')",
|
||||
`echo "$(jq -r '.selectedWorkspace.state' snapshot.json)"`,
|
||||
"value=$(jq -r '.revision.state' snapshot.json)",
|
||||
`echo "\`jq -r '.workspaceRevision.state' snapshot.json\`"`,
|
||||
`echo "$(cat <(jq -r '.selectedWorkspace.state' snapshot.json))"`,
|
||||
`${"$(".repeat(33)}jq -r '.revision.state' snapshot.json${")".repeat(33)}`,
|
||||
`echo "$(( $(jq -r '.revision.state' snapshot.json) + 0 ))"`,
|
||||
"echo \"$(( `jq -r '.workspaceRevision.state' snapshot.json` + 0 ))\"",
|
||||
"echo `echo \\`jq -r '.selectedWorkspace.state' snapshot.json\\``",
|
||||
"echo \"`echo \\`jq -r '.revision.state' snapshot.json\\``\"",
|
||||
`${"$(( ".repeat(33)}$(jq -r '.workspaceRevision.state' snapshot.json)${" + 0 ))".repeat(33)}`,
|
||||
'old=${revision["state"]}',
|
||||
"old=${workspaceRevision[state]}",
|
||||
"old=${revision[state]:-missing}",
|
||||
"old=${workspaceRevision['state']:=missing}",
|
||||
"old=${selectedWorkspace[state]:1:2}",
|
||||
]) rejects(source, "scripts/policy.sh");
|
||||
const jqFilters = [
|
||||
".revision?.state", '.["revision"]?.["state"]', '."revision".state',
|
||||
'."workspaceRevision"."state"', "(.revision).state", "$revision.state",
|
||||
".revision | .state", "(.workspaceRevision | .state)",
|
||||
'.["revi" + "sion"].state', '.["revi" + ("sion")].state',
|
||||
".revision | . | .state", "(.workspaceRevision | (.) | .state)",
|
||||
".revision | select(.) | .state", ".workspaceRevision | {value:.state}",
|
||||
'.revision | ["state"]', '(.workspaceRevision | (["state"]))', ".selectedWorkspace | $state",
|
||||
];
|
||||
for (const filter of jqFilters) {
|
||||
const compiled = spawnSync("jq", ["-n", "--argjson", "revision", "{}", "--arg", "state", "x", filter], { encoding: "utf8" });
|
||||
if (compiled.error?.code !== "ENOENT") assert.equal(compiled.status, 0, `${filter}: ${compiled.stderr}`);
|
||||
}
|
||||
passes("cat <<'EOF'\nrevision.state\nEOF\n", "scripts/literal.sh");
|
||||
passes("echo '${revision[state]}'\n", "scripts/single-quoted-parameter.sh");
|
||||
passes(`echo "<(jq '.revision.state')"\n`, "scripts/literal-process-text.sh");
|
||||
passes(`echo "ordinary jq '.workspaceRevision.state' text"\n`, "scripts/literal-jq-text.sh");
|
||||
passes(`echo '$(jq -r ".selectedWorkspace.state")'\n`, "scripts/single-quoted-command-text.sh");
|
||||
passes(`# profile's harmless note
|
||||
printf 'ok\n'
|
||||
`, "scripts/comment-apostrophe.sh");
|
||||
passes(`cat <( # profile's harmless note
|
||||
printf 'snapshot\n'
|
||||
)
|
||||
`, "scripts/substitution-comment-apostrophe.sh");
|
||||
passes(`echo "$(( 1 + (2 * 3) ))"\n`, "scripts/literal-arithmetic.sh");
|
||||
passes(`echo $(( jq + revision + state ))\n`, "scripts/arithmetic-identifiers.sh");
|
||||
passes("echo \\`jq -r '.revision.state' snapshot.json\\`\n", "scripts/escaped-literal-backticks.sh");
|
||||
passes("echo \"\\`jq -r '.workspaceRevision.state' snapshot.json\\`\"\n", "scripts/double-quoted-literal-backticks.sh");
|
||||
passes("echo `printf '%s' '\\`jq -r \".selectedWorkspace.state\" snapshot.json\\`'`\n", "scripts/quoted-nonexecuting-nested-backticks.sh");
|
||||
passes("jq --arg note 'revision.state' '.' file\n", "scripts/jq-arg.sh");
|
||||
passes(`jq --argjson note '"revision.state"' '.' file
|
||||
`, "scripts/jq-argjson.sh");
|
||||
passes("jq -r '.' revision.state.json\n", "scripts/jq-file.sh");
|
||||
passes("jq -r '.revision.id' snapshot.json\n", "scripts/jq-simple-non-state.sh");
|
||||
passes("jq -f revision.state.jq snapshot.json\n", "scripts/jq-from-file.sh");
|
||||
passes("jq --from-file workspaceRevision.state.jq snapshot.json\n", "scripts/jq-long-from-file.sh");
|
||||
passes(`jq -r '"revision.state"' snapshot.json
|
||||
`, "scripts/jq-string.sh");
|
||||
passes(`jq -r '{note:"selectedWorkspace.state"}' snapshot.json
|
||||
`, "scripts/jq-object.sh");
|
||||
passes(`jq -r '.revision | "state"' snapshot.json
|
||||
`, "scripts/jq-pipe-literal-right.sh");
|
||||
passes(`jq -r '"revision" | .state' snapshot.json
|
||||
`, "scripts/jq-pipe-literal-left.sh");
|
||||
passes(`jq -r '.revision | ["state"]' snapshot.json
|
||||
`, "scripts/jq-pipe-array.sh");
|
||||
passes(`jq -r '(.workspaceRevision | (["state"]))' snapshot.json
|
||||
`, "scripts/jq-pipe-parenthesized-array.sh");
|
||||
passes(`jq --arg state x '.selectedWorkspace | $state' snapshot.json
|
||||
`, "scripts/jq-pipe-variable.sh");
|
||||
for (const opener of ["'E'OF", "E'OF'", "E\\OF"]) {
|
||||
passes(`cat <<${opener}
|
||||
revision.state
|
||||
EOF
|
||||
`, "scripts/partial-quoted-heredoc.sh");
|
||||
}
|
||||
rejects("cat <<'E'OF\nrevision.state\nEOF\nworkspaceRevision.state\n", "scripts/after-heredoc.sh");
|
||||
rejects("cat <<'EOF'\nrevision.state\n", "scripts/unclosed-heredoc.sh");
|
||||
rejects(`echo "<<'EOF'"
|
||||
jq -r '.revision.state' snapshot.json
|
||||
`, "scripts/quoted-opener.sh");
|
||||
});
|
||||
|
||||
test("Python helper resolves active AST expressions and static format bindings", () => {
|
||||
const rejectsPython = (source) => assert.throws(
|
||||
() => validatePythonRevisionStates([{ source, label: "backend/scripts/policy.py" }]),
|
||||
/revision-state/,
|
||||
);
|
||||
for (const source of [
|
||||
"old = revision.state",
|
||||
'old = workspaceRevision["state"]',
|
||||
'old = record["selectedWorkspace"].state',
|
||||
'old = f"{revision.state}"',
|
||||
'"{revision.state}".format(value)',
|
||||
'"{0.state}".format(revision)',
|
||||
'"{0[state]}".format(workspaceRevision)',
|
||||
'"{item.state}".format(item=selectedWorkspace)',
|
||||
'"{item[state]}".format_map({"item": revision})',
|
||||
'("{0.state}").format(revision)',
|
||||
'"{0:{1.state}}".format(value, revision)',
|
||||
'old = revision["st" + "ate"]',
|
||||
'old = record["revi" + "sion"].state',
|
||||
'old = revision[f"state"]',
|
||||
'old = record[f"revision"].state',
|
||||
`old = revision[f"st{'a'}te"]`,
|
||||
`old = revision[f"{'state'}"]`,
|
||||
`old = record[f"revi{'sion'}"].state`,
|
||||
`old = revision[f"{'st' + 'ate'}"]`,
|
||||
`old = record[f"{'revi' + 'sion'}"].state`,
|
||||
`old = revision[f"{'state':s}"]`,
|
||||
'"{0.state}".format(*[revision])',
|
||||
'"{0[state]}".format(*(revision,))',
|
||||
'"{1[state]}".format(*[other, workspaceRevision])',
|
||||
'"{item.state}".format(**{"item": selectedWorkspace})',
|
||||
'"{item[state]}".format_map({**{"item": revision}})',
|
||||
'"{.state}".format(revision)',
|
||||
'"{[state]}".format(revision)',
|
||||
'"{:{.state}}".format(value, revision)',
|
||||
'"{.name} {[state]}".format(other, revision)',
|
||||
'"{item.state}".format(item=revision, **values)',
|
||||
]) rejectsPython(source);
|
||||
validatePythonRevisionStates([
|
||||
{ source: 'text = "{revision.state}"', label: "backend/scripts/literal.py" },
|
||||
{ source: 'text = "{{revision.state}}".format(value)', label: "backend/scripts/escaped.py" },
|
||||
{ source: 'text = "{0.state}".format(other)', label: "backend/scripts/unrelated.py" },
|
||||
{ source: 'old = revision[f"st{suffix}"]', label: "backend/scripts/dynamic-key.py" },
|
||||
{ source: 'text = "{.name} {[state]}".format(other, other)', label: "backend/scripts/multi-auto.py" },
|
||||
{ source: 'text = "{item.state}".format(**values)', label: "backend/scripts/dynamic-map.py" },
|
||||
]);
|
||||
const hostile = mkdtempSync(join(tmpdir(), "revision-policy-hostile-"));
|
||||
writeFileSync(join(hostile, "json.py"), "raise RuntimeError('shadowed')\n");
|
||||
const previousPythonPath = process.env.PYTHONPATH;
|
||||
try {
|
||||
process.env.PYTHONPATH = hostile;
|
||||
validatePythonRevisionStates([{ source: "value = 1", label: "backend/scripts/isolated.py" }]);
|
||||
} finally {
|
||||
if (previousPythonPath === undefined) delete process.env.PYTHONPATH;
|
||||
else process.env.PYTHONPATH = previousPythonPath;
|
||||
rmSync(hostile, { recursive: true, force: true });
|
||||
}
|
||||
assert.throws(
|
||||
() => validatePythonRevisionStates([{ source: 'revision[f"{1:.1000000000f}"]', label: "backend/scripts/oversized.py" }]),
|
||||
/revision-state helper failed/,
|
||||
);
|
||||
validatePythonRevisionStates([{ source: 'revision[f"{1:04d}"]', label: "backend/scripts/small-format.py" }]);
|
||||
assert.throws(
|
||||
() => validatePythonRevisionStates([{ source: "def broken(", label: "backend/scripts/invalid.py" }]),
|
||||
/revision-state helper failed/,
|
||||
);
|
||||
});
|
||||
|
||||
test("YAML mappings and only active plain scalar expressions are rejected", () => {
|
||||
for (const source of [
|
||||
"value: { revision: { state: old } }\n",
|
||||
"value:\n workspaceRevision:\n state: old\n",
|
||||
'items:\n - "selectedWorkspace":\n "state": old\n',
|
||||
"old: selectedWorkspace.state\n",
|
||||
"url: https://host/x; old: selectedWorkspace.state\n",
|
||||
"saved: &saved { state: old }\nvalue: { revision: *saved }\n",
|
||||
"defaults: &defaults { workspaceRevision: { state: old } }\nvalue: { <<: *defaults }\n",
|
||||
]) rejects(source, "scripts/policy.yaml");
|
||||
rejects("a: &a [x,x,x,x,x,x,x,x,x]\nb: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]\nc: [*b,*b,*b,*b,*b,*b,*b,*b,*b]\n", "scripts/alias-bomb.yaml");
|
||||
rejects("value: [\n", "scripts/invalid.yaml");
|
||||
for (const source of [
|
||||
"value: |\n revision.state\n",
|
||||
"value: >\n workspaceRevision.state\n",
|
||||
'value: "selectedWorkspace.state"\n',
|
||||
"url: https://host/revision.state\n",
|
||||
]) passes(source, "scripts/literal.yaml");
|
||||
});
|
||||
@@ -1,318 +0,0 @@
|
||||
"""Semantic Python revision-state policy helper.
|
||||
|
||||
Reads one JSON array of ``{"label": str, "source": str}`` records from stdin and
|
||||
writes ``{"violations": [label, ...]}``. Invalid input or Python source is fatal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import string
|
||||
import sys
|
||||
from itertools import pairwise
|
||||
from typing import Any
|
||||
|
||||
TARGETS = frozenset({"revision", "workspaceRevision", "selectedWorkspace"})
|
||||
_FORMATTER = string.Formatter()
|
||||
|
||||
|
||||
MAX_STATIC_TEXT = 4_096
|
||||
MAX_FORMAT_SPEC = 256
|
||||
MAX_STATIC_DEPTH = 64
|
||||
_UNRESOLVED = object()
|
||||
|
||||
|
||||
def _bounded_text(value: str) -> str:
|
||||
if len(value) > MAX_STATIC_TEXT:
|
||||
raise ValueError("static text exceeds revision policy limit")
|
||||
return value
|
||||
|
||||
|
||||
def _static_scalar(node: ast.expr, depth: int) -> object:
|
||||
if depth > MAX_STATIC_DEPTH:
|
||||
raise ValueError("static expression nesting exceeds revision policy limit")
|
||||
if isinstance(node, ast.Constant) and type(node.value) in {
|
||||
str,
|
||||
int,
|
||||
float,
|
||||
complex,
|
||||
bool,
|
||||
type(None),
|
||||
}:
|
||||
if isinstance(node.value, str):
|
||||
_bounded_text(node.value)
|
||||
if isinstance(node.value, int) and node.value.bit_length() > MAX_STATIC_TEXT * 4:
|
||||
raise ValueError("static integer exceeds revision policy limit")
|
||||
return node.value
|
||||
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
|
||||
left = _static_scalar(node.left, depth + 1)
|
||||
right = _static_scalar(node.right, depth + 1)
|
||||
if left is _UNRESOLVED or right is _UNRESOLVED:
|
||||
return _UNRESOLVED
|
||||
try:
|
||||
result = left + right
|
||||
except TypeError:
|
||||
return _UNRESOLVED
|
||||
if type(result) not in {str, int, float, complex, bool}:
|
||||
return _UNRESOLVED
|
||||
if isinstance(result, str):
|
||||
_bounded_text(result)
|
||||
if isinstance(result, int) and result.bit_length() > MAX_STATIC_TEXT * 4:
|
||||
raise ValueError("static integer exceeds revision policy limit")
|
||||
return result
|
||||
if isinstance(node, ast.JoinedStr):
|
||||
result = _static_key(node, depth + 1)
|
||||
return _UNRESOLVED if result is None else result
|
||||
return _UNRESOLVED
|
||||
|
||||
|
||||
def _validate_format_spec(format_spec: str) -> None:
|
||||
if len(format_spec) > MAX_FORMAT_SPEC:
|
||||
raise ValueError("static format specification exceeds revision policy limit")
|
||||
for digits in re.findall(r"[0-9]+", format_spec):
|
||||
if len(digits) > 6 or int(digits) > MAX_STATIC_TEXT:
|
||||
raise ValueError("static format width or precision exceeds revision policy limit")
|
||||
|
||||
|
||||
def _static_key(node: ast.expr, depth: int = 0) -> str | None:
|
||||
if depth > MAX_STATIC_DEPTH:
|
||||
raise ValueError("static key nesting exceeds revision policy limit")
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
return _bounded_text(node.value)
|
||||
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
|
||||
left = _static_key(node.left, depth + 1)
|
||||
right = _static_key(node.right, depth + 1)
|
||||
return None if left is None or right is None else _bounded_text(left + right)
|
||||
if isinstance(node, ast.JoinedStr):
|
||||
pieces = []
|
||||
length = 0
|
||||
for value in node.values:
|
||||
if isinstance(value, ast.Constant) and isinstance(value.value, str):
|
||||
piece = value.value
|
||||
elif isinstance(value, ast.FormattedValue):
|
||||
scalar = _static_scalar(value.value, depth + 1)
|
||||
if scalar is _UNRESOLVED:
|
||||
return None
|
||||
format_spec = "" if value.format_spec is None else _static_key(value.format_spec, depth + 1)
|
||||
if format_spec is None:
|
||||
return None
|
||||
_validate_format_spec(format_spec)
|
||||
try:
|
||||
if value.conversion == ord("s"):
|
||||
scalar = str(scalar)
|
||||
elif value.conversion == ord("r"):
|
||||
scalar = repr(scalar)
|
||||
elif value.conversion == ord("a"):
|
||||
scalar = ascii(scalar)
|
||||
elif value.conversion != -1:
|
||||
return None
|
||||
piece = format(scalar, format_spec)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
length += len(piece)
|
||||
if length > MAX_STATIC_TEXT:
|
||||
raise ValueError("static formatted key exceeds revision policy limit")
|
||||
pieces.append(piece)
|
||||
return "".join(pieces)
|
||||
return None
|
||||
|
||||
|
||||
def _is_revision_expr(node: ast.expr) -> bool:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id in TARGETS
|
||||
if isinstance(node, ast.Attribute):
|
||||
return node.attr in TARGETS
|
||||
if isinstance(node, ast.Subscript):
|
||||
return _static_key(node.slice) in TARGETS
|
||||
return False
|
||||
|
||||
|
||||
def _is_state_access(node: ast.AST) -> bool:
|
||||
if isinstance(node, ast.Attribute):
|
||||
return node.attr == "state" and _is_revision_expr(node.value)
|
||||
if isinstance(node, ast.Subscript):
|
||||
return _static_key(node.slice) == "state" and _is_revision_expr(node.value)
|
||||
return False
|
||||
|
||||
|
||||
def _static_sequence(node: ast.expr) -> list[ast.expr] | None:
|
||||
if not isinstance(node, (ast.List, ast.Tuple)):
|
||||
return None
|
||||
result: list[ast.expr] = []
|
||||
for element in node.elts:
|
||||
if isinstance(element, ast.Starred):
|
||||
nested = _static_sequence(element.value)
|
||||
if nested is None:
|
||||
return None
|
||||
result.extend(nested)
|
||||
else:
|
||||
result.append(element)
|
||||
return result
|
||||
|
||||
|
||||
def _static_mapping(node: ast.expr) -> dict[str, ast.expr] | None:
|
||||
if not isinstance(node, ast.Dict):
|
||||
return None
|
||||
result: dict[str, ast.expr] = {}
|
||||
for key, value in zip(node.keys, node.values, strict=True):
|
||||
if key is None:
|
||||
nested = _static_mapping(value)
|
||||
if nested is None:
|
||||
return None
|
||||
result.update(nested)
|
||||
elif (name := _static_key(key)) is not None:
|
||||
result[name] = value
|
||||
else:
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
def _format_bindings(call: ast.Call, method: str) -> dict[str | int, ast.expr]:
|
||||
if method == "format":
|
||||
bindings: dict[str | int, ast.expr] = {}
|
||||
position = 0
|
||||
positional_known = True
|
||||
for argument in call.args:
|
||||
if isinstance(argument, ast.Starred):
|
||||
expanded = _static_sequence(argument.value)
|
||||
if expanded is None:
|
||||
positional_known = False
|
||||
continue
|
||||
if positional_known:
|
||||
for value in expanded:
|
||||
bindings[position] = value
|
||||
position += 1
|
||||
elif positional_known:
|
||||
bindings[position] = argument
|
||||
position += 1
|
||||
for keyword in call.keywords:
|
||||
if keyword.arg is not None:
|
||||
# An explicit keyword remains bound even beside **dynamic; a duplicate is TypeError.
|
||||
bindings[keyword.arg] = keyword.value
|
||||
else:
|
||||
expanded = _static_mapping(keyword.value)
|
||||
if expanded is not None:
|
||||
bindings.update(expanded)
|
||||
return bindings
|
||||
if len(call.args) != 1 or call.keywords:
|
||||
return {}
|
||||
return _static_mapping(call.args[0]) or {}
|
||||
|
||||
|
||||
def _field_accesses_state(
|
||||
field_name: str, bindings: dict[str | int, ast.expr], automatic_index: int | None = None
|
||||
) -> bool:
|
||||
root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name)
|
||||
if root_match is None:
|
||||
if automatic_index is None or not field_name.startswith((".", "[")):
|
||||
return False
|
||||
root: str | int = automatic_index
|
||||
cursor = 0
|
||||
else:
|
||||
root_text = root_match.group(0)
|
||||
root = int(root_text) if root_text.isdigit() else root_text
|
||||
cursor = root_match.end()
|
||||
steps: list[tuple[bool, str]] = []
|
||||
while cursor < len(field_name):
|
||||
if field_name[cursor] == ".":
|
||||
match = re.match(r"[A-Za-z_][A-Za-z0-9_]*", field_name[cursor + 1 :])
|
||||
if match is None:
|
||||
return False
|
||||
steps.append((True, match.group(0)))
|
||||
cursor += len(match.group(0)) + 1
|
||||
elif field_name[cursor] == "[":
|
||||
close = field_name.find("]", cursor + 1)
|
||||
if close < 0:
|
||||
return False
|
||||
steps.append((False, field_name[cursor + 1 : close]))
|
||||
cursor = close + 1
|
||||
else:
|
||||
return False
|
||||
if steps:
|
||||
first_step = str(steps[0][1])
|
||||
if str(root) in TARGETS and first_step == "state":
|
||||
return True
|
||||
bound = bindings.get(root)
|
||||
if bound is not None and _is_revision_expr(bound) and first_step == "state":
|
||||
return True
|
||||
names = [str(root), *(str(key) for _is_attr, key in steps)]
|
||||
return any(left in TARGETS and right == "state" for left, right in pairwise(names))
|
||||
|
||||
|
||||
def _format_call_violation(node: ast.Call) -> bool:
|
||||
function = node.func
|
||||
if not isinstance(function, ast.Attribute) or function.attr not in {"format", "format_map"}:
|
||||
return False
|
||||
if not isinstance(function.value, ast.Constant) or not isinstance(function.value.value, str):
|
||||
return False
|
||||
bindings = _format_bindings(node, function.attr)
|
||||
numbering: dict[str, int | str | None] = {"next": 0, "mode": None}
|
||||
visited = 0
|
||||
|
||||
def analyze_template(template: str) -> bool:
|
||||
nonlocal visited
|
||||
visited += 1
|
||||
if visited > 1_000:
|
||||
raise ValueError("format specification nesting exceeds policy limit")
|
||||
for _literal, field_name, format_spec, _conversion in _FORMATTER.parse(template):
|
||||
automatic_index = None
|
||||
if field_name is not None:
|
||||
root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name)
|
||||
automatic = field_name == "" or root_match is None and field_name.startswith((".", "["))
|
||||
manual = root_match is not None and root_match.group(0).isdigit()
|
||||
if automatic:
|
||||
if numbering["mode"] == "manual":
|
||||
raise ValueError("cannot switch from manual to automatic field numbering")
|
||||
numbering["mode"] = "automatic"
|
||||
automatic_index = int(numbering["next"])
|
||||
numbering["next"] = automatic_index + 1
|
||||
elif manual:
|
||||
if numbering["mode"] == "automatic":
|
||||
raise ValueError("cannot switch from automatic to manual field numbering")
|
||||
numbering["mode"] = "manual"
|
||||
if _field_accesses_state(field_name, bindings, automatic_index):
|
||||
return True
|
||||
if format_spec and analyze_template(format_spec):
|
||||
return True
|
||||
return False
|
||||
|
||||
return analyze_template(function.value.value)
|
||||
|
||||
|
||||
def has_revision_state(source: str, label: str = "<unknown>") -> bool:
|
||||
tree = ast.parse(source, filename=label, mode="exec")
|
||||
return any(_is_state_access(node) or (isinstance(node, ast.Call) and _format_call_violation(node)) for node in ast.walk(tree))
|
||||
|
||||
|
||||
def analyze_batch(records: Any) -> list[str]:
|
||||
if not isinstance(records, list):
|
||||
raise TypeError("input must be a JSON array")
|
||||
violations = []
|
||||
for record in records:
|
||||
if not isinstance(record, dict) or set(record) != {"label", "source"}:
|
||||
raise TypeError("each record must contain exactly label and source")
|
||||
label, source = record["label"], record["source"]
|
||||
if not isinstance(label, str) or not isinstance(source, str):
|
||||
raise TypeError("label and source must be strings")
|
||||
if has_revision_state(source, label):
|
||||
violations.append(label)
|
||||
return violations
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
records = json.load(sys.stdin)
|
||||
json.dump({"violations": analyze_batch(records)}, sys.stdout, ensure_ascii=False)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
except Exception as error: # noqa: BLE001 - protocol boundary must fail closed
|
||||
print(f"python revision-state helper failed: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,6 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const path = process.env.THT_SSH_PASSPHRASE_FILE;
|
||||
if (!path) process.exit(1);
|
||||
process.stdout.write(readFileSync(path));
|
||||
@@ -1,90 +0,0 @@
|
||||
import importlib.util
|
||||
import tracemalloc
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
_HELPER = Path(__file__).with_name("revision_state_policy.py")
|
||||
_SPEC = importlib.util.spec_from_file_location("revision_state_policy", _HELPER)
|
||||
assert _SPEC is not None and _SPEC.loader is not None
|
||||
_MODULE = importlib.util.module_from_spec(_SPEC)
|
||||
_SPEC.loader.exec_module(_MODULE)
|
||||
analyze_batch = _MODULE.analyze_batch
|
||||
has_revision_state = _MODULE.has_revision_state
|
||||
|
||||
|
||||
class RevisionStatePolicyTests(unittest.TestCase):
|
||||
def test_ast_access_and_f_strings(self):
|
||||
for source in (
|
||||
"old = revision.state",
|
||||
'old = workspaceRevision["state"]',
|
||||
'old = record["selectedWorkspace"].state',
|
||||
'old = f"{revision.state}"',
|
||||
'old = revision["st" + "ate"]',
|
||||
'old = record["revi" + "sion"].state',
|
||||
'old = revision[f"state"]',
|
||||
'old = record[f"revision"].state',
|
||||
"old = revision[f\"st{'a'}te\"]",
|
||||
"old = revision[f\"{'state'}\"]",
|
||||
"old = record[f\"revi{'sion'}\"].state",
|
||||
"old = revision[f\"{'st' + 'ate'}\"]",
|
||||
"old = record[f\"{'revi' + 'sion'}\"].state",
|
||||
"old = revision[f\"{'state':s}\"]",
|
||||
):
|
||||
with self.subTest(source=source):
|
||||
self.assertTrue(has_revision_state(source))
|
||||
|
||||
def test_static_format_bindings(self):
|
||||
for source in (
|
||||
'"{0.state}".format(revision)',
|
||||
'"{0[state]}".format(workspaceRevision)',
|
||||
'"{item.state}".format(item=selectedWorkspace)',
|
||||
'"{item[state]}".format_map({"item": revision})',
|
||||
'("{0.state}").format(revision)',
|
||||
'"{0:{1.state}}".format(value, revision)',
|
||||
'"{0.state}".format(*[revision])',
|
||||
'"{0[state]}".format(*(revision,))',
|
||||
'"{1[state]}".format(*[other, workspaceRevision])',
|
||||
'"{item.state}".format(**{"item": selectedWorkspace})',
|
||||
'"{item[state]}".format(**{"outer": other, **{"item": revision}})',
|
||||
'"{item.state}".format_map({**{"item": workspaceRevision}})',
|
||||
'"{.state}".format(revision)',
|
||||
'"{[state]}".format(revision)',
|
||||
'"{:{.state}}".format(value, revision)',
|
||||
'"{.name} {[state]}".format(other, revision)',
|
||||
'"{item.state}".format(item=revision, **values)',
|
||||
):
|
||||
with self.subTest(source=source):
|
||||
self.assertTrue(has_revision_state(source))
|
||||
self.assertFalse(has_revision_state('"{0.state}".format(other)'))
|
||||
# Dynamic unpacking is intentionally unresolved rather than guessed.
|
||||
self.assertFalse(has_revision_state('"{0.state}".format(*values)'))
|
||||
self.assertFalse(has_revision_state('"{.name} {[state]}".format(other, other)'))
|
||||
self.assertFalse(has_revision_state('"{item.state}".format(**values)'))
|
||||
# FormattedValue keys are dynamic and are not treated as static strings.
|
||||
self.assertFalse(has_revision_state('revision[f"st{suffix}"]'))
|
||||
|
||||
def test_literals_are_not_active(self):
|
||||
self.assertFalse(has_revision_state('text = "{revision.state}"'))
|
||||
self.assertFalse(has_revision_state('text = "{{revision.state}}".format(value)'))
|
||||
|
||||
def test_oversized_static_format_fails_before_formatting(self):
|
||||
tracemalloc.start()
|
||||
with patch("builtins.format") as format_mock:
|
||||
with self.assertRaisesRegex(ValueError, "width or precision"):
|
||||
has_revision_state('revision[f"{1:.1000000000f}"]')
|
||||
format_mock.assert_not_called()
|
||||
_current, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
self.assertLess(peak, 1_000_000)
|
||||
self.assertFalse(has_revision_state('revision[f"{1:04d}"]'))
|
||||
|
||||
def test_batch_contract(self):
|
||||
self.assertEqual(
|
||||
analyze_batch([{"label": "one.py", "source": "revision.state"}]),
|
||||
["one.py"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,378 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash } from "node:crypto";
|
||||
import { lstat, readFile, realpath } from "node:fs/promises";
|
||||
import { isAbsolute, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
import { isMap, isScalar, parseAllDocuments } from "yaml";
|
||||
import { extractBashDocuments } from "./bash-heredoc.mjs";
|
||||
import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs";
|
||||
import { parseWorkspaceYaml } from "../dist/workspaces/schema.js";
|
||||
|
||||
const scriptPath = fileURLToPath(import.meta.url);
|
||||
const allowedKinds = new Set(["policy_text", "workspace_descriptor", "deployment_script"]);
|
||||
// Exact-content trust exceptions. Each digest covers the raw UTF-8 bytes from the
|
||||
// opener line through the closer line (including physical line endings). These
|
||||
// blocks are reviewed non-workspace runtime/config generation, not semantic proof.
|
||||
const reviewedExpandableBlocks = new Map([
|
||||
["scripts/test-dwh-auth-nginx-integration.sh", [
|
||||
{ sha256: "ead57234ad3520b5c7d4262b772957cbc7b9589da4f35fb17b160f948eb2ac7b", rationale: "Generates the reviewed isolated Nginx integration configuration." },
|
||||
]],
|
||||
["scripts/test-install-tht.sh", [
|
||||
{ sha256: "37f18ce7ce93cb8b84f3b3708462cc16d50fdc7bab22836c382dbacf8382f05f", rationale: "Generates the reviewed synthetic tht installer artifact." },
|
||||
]],
|
||||
["scripts/test-server-pi-state-topology.sh", [
|
||||
{ sha256: "6ae9567db53d6cd45a2c19c98acaf45f382450b157ea7d6f6d35125f68c50947", rationale: "Generates the isolated server topology test environment, including its installation descriptor and authentication configuration root." },
|
||||
]],
|
||||
["scripts/test-vector-backup-restore-safety.sh", [
|
||||
{ sha256: "40b8a10a3c06aaa98e324fbf688b7d1f5cead330d7ba7eef98e06256d412a85a", rationale: "Generates the reviewed restore safety manifest." },
|
||||
]],
|
||||
["scripts/test-windows-clone-contract.ps1", [
|
||||
{ sha256: "3204f772d33cad42bcac99191507051aefb2c91d2935bec6698b956e44f9bf45", rationale: "Generates reviewed Windows clone test configuration with its authentication configuration root." },
|
||||
{ sha256: "f4814d842a7502b7ef30fd6b224d5cb17b0ffd6fb2367c41c49ac16587536d93", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
|
||||
{ sha256: "6f25ce3b58cea47b74fe9319ed917d8089a2fb334bc0d469daa7e1f10865d870", rationale: "Generates the reviewed Windows Compose override for the canonical service topology." },
|
||||
{ sha256: "45a3cf19f7ce697b858b63d27a4edc7fefa2414d0408e7b6d72a65c86d314f5b", rationale: "Same reviewed Compose override in the repository-required CRLF checkout representation." },
|
||||
{ sha256: "5d0d1a3fc45e99b3aacaf4ee5dd09a6bee1937784375dfe4bcfaa4ae32cfb9de", rationale: "Generates reviewed Windows clone test configuration." },
|
||||
{ sha256: "b903e5dae953ae1372f1a5276f12a92ed3dd632b897f3afe5e00c646d90a1b42", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
|
||||
]],
|
||||
["scripts/unified-deployment-smoke.sh", [
|
||||
{ sha256: "1d60bf140165a8fabfa0c3729e776136904717e67becf3e0ab68c70d8e37847e", rationale: "Generates reviewed Task 13 runtime configuration." },
|
||||
{ sha256: "36d3d8a2362dbdc4fad90948d6c227586d749f56b9a4bc5b6b5a91bcbec6407b", rationale: "Generates the reviewed local Task 13 Compose override." },
|
||||
{ sha256: "c556f7d910d0788e219b042957e6b307cb9925b43920c680535d0d3a6dcbdb25", rationale: "Generates the reviewed local Task 13 installation descriptor." },
|
||||
{ sha256: "526006fa6d48a8080b3834723630c64de5005a67243e944ebf1da15212b4d654", rationale: "Generates the reviewed server Task 13 Compose override." },
|
||||
{ sha256: "c57ae2205c21ead0c2015a353aaabb948fa4ddd9b78a2cdcdb71f48cf2db742d", rationale: "Generates the reviewed projected-auth server Task 13 installation descriptor." },
|
||||
]],
|
||||
["scripts/vector-backup.sh", [
|
||||
{ sha256: "571899db49dfdcec8107fbe1e0a86a61e7581979d3c4c248c20546843e275bcf", rationale: "Generates the reviewed backup manifest inside the helper command." },
|
||||
]],
|
||||
["scripts/vector-restore.sh", [
|
||||
{ sha256: "f04d872e556a7323583c6e620b25814fb6a8e2568a9a555623978185b473a49d", rationale: "Feeds reviewed parsed manifest values to read loops." },
|
||||
{ sha256: "c6053ed44abae71ae4821b68f9a513f8070947350e30d89ae0f65bf4a48f66fd", rationale: "Feeds reviewed parsed manifest values to read loops." },
|
||||
]],
|
||||
]);
|
||||
|
||||
function blockDigest(rawBlock) {
|
||||
return createHash("sha256").update(rawBlock, "utf8").digest("hex");
|
||||
}
|
||||
|
||||
function reviewedExpandableBlock(path, rawBlock) {
|
||||
const digest = blockDigest(rawBlock);
|
||||
return (reviewedExpandableBlocks.get(path) ?? []).some((review) => review.sha256 === digest);
|
||||
}
|
||||
|
||||
function hasAmbiguousExpansion(source, path) {
|
||||
const powershell = path.endsWith(".ps1");
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
const character = source[index];
|
||||
if (powershell && character === "`") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (!powershell && character === "\\") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "$" || (!powershell && character === "`")) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function physicalLines(source) {
|
||||
const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? [];
|
||||
if (rawLines.length === 0) rawLines.push("");
|
||||
return rawLines.map((raw) => ({ raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, "") }));
|
||||
}
|
||||
const prescribedSymbols = [
|
||||
"WorkspaceV1", "WorkspaceV2", "DeprecatedV2Descriptor", "LegacyMigrationResult",
|
||||
"LegacyMigrationOptions", "WorkspaceV2MigrationInput", "migrateLegacyWorkspace",
|
||||
"writeMigratedWorkspace", "migrateWorkspaceV1ToV2", "migrateWorkspaceV2ToV3",
|
||||
];
|
||||
const migrationMarkers = ["migration_required", "deprecated-v2-descriptor", "migrate-legacy", "migrate-v2-qdrant"];
|
||||
|
||||
function isPolicyImplementationException(label, category) {
|
||||
const implementations = new Set([
|
||||
"scripts/verify-schema-v3-only.sh",
|
||||
"scripts/test-verify-schema-v3-only.sh",
|
||||
"backend/scripts/verify-workspace-descriptor-files.mjs",
|
||||
"backend/scripts/verify-workspace-descriptor-files.test.mjs",
|
||||
"backend/scripts/revision-state-policy.mjs",
|
||||
"backend/scripts/revision-state-policy.test.mjs",
|
||||
"backend/scripts/bash-heredoc.mjs",
|
||||
"backend/scripts/revision_state_policy.py",
|
||||
"backend/scripts/test_revision_state_policy.py",
|
||||
]);
|
||||
if (implementations.has(label)) return true;
|
||||
if (category === "migration-marker" && new Set([
|
||||
"scripts/workspace_descriptor_doc_contract.py",
|
||||
"scripts/test_workspace_descriptor_doc_contract.py",
|
||||
"backend/scripts/clean-dist.test.mjs",
|
||||
]).has(label)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
function validatePolicySource(source, label) {
|
||||
if (!isPolicyImplementationException(label, "prescribed-symbol")) {
|
||||
for (const symbol of prescribedSymbols) {
|
||||
if (source.toLowerCase().includes(symbol.toLowerCase())) throw new Error(`${label}: forbidden prescribed-symbol substring: ${symbol}`);
|
||||
}
|
||||
}
|
||||
if (!isPolicyImplementationException(label, "migration-marker")) {
|
||||
for (const marker of migrationMarkers) {
|
||||
if (source.toLowerCase().includes(marker.toLowerCase())) throw new Error(`${label}: forbidden migration-marker substring: ${marker}`);
|
||||
}
|
||||
}
|
||||
if (!isPolicyImplementationException(label, "legacy-workspace")) {
|
||||
for (const match of source.matchAll(/legacyworkspace/giu)) {
|
||||
if (match[0] !== "legacyWorkspace") throw new Error(`${label}: forbidden legacy-workspace spelling: ${match[0]}`);
|
||||
}
|
||||
}
|
||||
if (!/\.pyw?$/iu.test(label) && !isPolicyImplementationException(label, "revision-state")) validateRevisionState(source, label);
|
||||
}
|
||||
|
||||
function documentShape(document) {
|
||||
const shape = { workspacePresent: false, workspaceMapping: false };
|
||||
if (!isMap(document.contents)) return shape;
|
||||
for (const pair of document.contents.items) {
|
||||
if (!isScalar(pair.key)) continue;
|
||||
if (pair.key.value === "workspace") {
|
||||
shape.workspacePresent = true;
|
||||
if (isMap(pair.value)) shape.workspaceMapping = true;
|
||||
}
|
||||
}
|
||||
return shape;
|
||||
}
|
||||
|
||||
function documents(source) {
|
||||
try {
|
||||
return parseAllDocuments(source, { uniqueKeys: true });
|
||||
} catch (error) {
|
||||
throw new Error(`YAML parser failed: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateWorkspaceSource(source, label, { requireWorkspace, expandable = false, path, rawBlock }) {
|
||||
const parsed = documents(source);
|
||||
const shapes = parsed.map(documentShape);
|
||||
if (requireWorkspace) {
|
||||
if (!shapes.some((shape) => shape.workspacePresent)) {
|
||||
throw new Error(`${label}: expected a top-level workspace mapping`);
|
||||
}
|
||||
if (!shapes.some((shape) => shape.workspaceMapping)) {
|
||||
throw new Error(`${label}: top-level workspace must be a mapping`);
|
||||
}
|
||||
} else {
|
||||
if (expandable && hasAmbiguousExpansion(source, path) && !reviewedExpandableBlock(path, rawBlock)) {
|
||||
throw new Error(`${label}: expandable block interpolation is not in the exact-content reviewed allowlist`);
|
||||
}
|
||||
if (shapes.some((shape) => shape.workspaceMapping)) {
|
||||
throw new Error(`${label}: embedded workspace descriptor is forbidden; use a tracked workspace fixture`);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
parseWorkspaceYaml(source);
|
||||
} catch (error) {
|
||||
throw new Error(`${label}: workspace descriptor is not valid schema v3: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function deploymentScriptDialect(path) {
|
||||
if (path.endsWith(".sh")) return "bash";
|
||||
if (path.endsWith(".ps1")) return "powershell";
|
||||
throw new Error(`${path}: unknown deployment script dialect`);
|
||||
}
|
||||
|
||||
|
||||
function powerShellHereStringOpener(line, state) {
|
||||
let quote = null;
|
||||
for (let index = 0; index < line.length; index += 1) {
|
||||
if (state.blockComment) {
|
||||
const close = line.indexOf("#>", index);
|
||||
if (close < 0) return null;
|
||||
state.blockComment = false;
|
||||
index = close + 1;
|
||||
continue;
|
||||
}
|
||||
const character = line[index];
|
||||
if (quote === null && character === "`") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (quote === "'") {
|
||||
if (character === "'" && line[index + 1] === "'") index += 1;
|
||||
else if (character === "'") quote = null;
|
||||
continue;
|
||||
}
|
||||
if (quote === '"') {
|
||||
if (character === "`") index += 1;
|
||||
else if (character === '"') quote = null;
|
||||
continue;
|
||||
}
|
||||
if (character === "#") return null;
|
||||
if (character === "<" && line[index + 1] === "#") {
|
||||
state.blockComment = true;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "@" && (line[index + 1] === "'" || line[index + 1] === '"') && /^[ \t]*$/u.test(line.slice(index + 2))) return line[index + 1];
|
||||
if (character === "'" || character === '"') quote = character;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractPowerShellDocuments(source, label) {
|
||||
const records = physicalLines(source);
|
||||
const lines = records.map((record) => record.text);
|
||||
const extracted = [];
|
||||
const state = { blockComment: false };
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const quote = powerShellHereStringOpener(lines[index], state);
|
||||
if (quote === null) continue;
|
||||
const delimiter = `${quote}@`;
|
||||
const opener = index;
|
||||
const body = [];
|
||||
const start = index + 2;
|
||||
let closed = false;
|
||||
for (index += 1; index < lines.length; index += 1) {
|
||||
if (lines[index].trimEnd() === delimiter) {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
body.push(lines[index]);
|
||||
}
|
||||
extracted.push({
|
||||
source: `${body.join("\n")}\n`,
|
||||
label: `${label}:${start} PowerShell here-string${closed ? "" : " (unclosed)"}`,
|
||||
expandable: quote === '"',
|
||||
path: label,
|
||||
rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""),
|
||||
});
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
|
||||
export function extractScriptDocuments(source, label = "deployment script") {
|
||||
const dialect = deploymentScriptDialect(label);
|
||||
if (dialect === "bash") return extractBashDocuments(source, label);
|
||||
return extractPowerShellDocuments(source, label);
|
||||
}
|
||||
|
||||
async function safeFile(root, path) {
|
||||
if (typeof path !== "string" || path.length === 0 || isAbsolute(path) || path.includes("\\")) {
|
||||
throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`);
|
||||
}
|
||||
const segments = path.split("/");
|
||||
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
|
||||
throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`);
|
||||
}
|
||||
const absolute = resolve(root, ...segments);
|
||||
const fromRoot = relative(root, absolute);
|
||||
if (fromRoot.startsWith(`..${sep}`) || fromRoot === ".." || isAbsolute(fromRoot)) {
|
||||
throw new Error(`verifier path escapes root: ${JSON.stringify(path)}`);
|
||||
}
|
||||
const entry = await lstat(absolute);
|
||||
if (!entry.isFile() || entry.isSymbolicLink()) {
|
||||
throw new Error(`verifier input is not a regular file: ${path}`);
|
||||
}
|
||||
const canonical = await realpath(absolute);
|
||||
const canonicalRelative = relative(root, canonical);
|
||||
if (canonicalRelative.startsWith(`..${sep}`) || canonicalRelative === ".." || isAbsolute(canonicalRelative)) {
|
||||
throw new Error(`verifier input resolves outside root: ${path}`);
|
||||
}
|
||||
return absolute;
|
||||
}
|
||||
|
||||
export async function verifyEntries({ root, entries }) {
|
||||
const canonicalRoot = await realpath(root);
|
||||
const seen = new Set();
|
||||
const pythonPolicies = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry || !allowedKinds.has(entry.kind) || typeof entry.path !== "string") {
|
||||
throw new Error("workspace verifier manifest contains an invalid entry");
|
||||
}
|
||||
const identity = `${entry.kind}\0${entry.path}`;
|
||||
if (seen.has(identity)) throw new Error(`workspace verifier manifest duplicates: ${entry.path}`);
|
||||
seen.add(identity);
|
||||
const absolute = await safeFile(canonicalRoot, entry.path);
|
||||
const bytes = await readFile(absolute);
|
||||
let source;
|
||||
try {
|
||||
source = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
} catch {
|
||||
throw new Error(`${entry.path}: input is not valid UTF-8`);
|
||||
}
|
||||
if (source.includes("\0")) throw new Error(`${entry.path}: NUL byte is forbidden`);
|
||||
if (entry.kind === "policy_text") {
|
||||
validatePolicySource(source, entry.path);
|
||||
if (/\.pyw?$/iu.test(entry.path) && !isPolicyImplementationException(entry.path, "revision-state")) {
|
||||
pythonPolicies.push({ label: entry.path, source });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (entry.kind === "workspace_descriptor") {
|
||||
validateWorkspaceSource(source, entry.path, { requireWorkspace: true });
|
||||
continue;
|
||||
}
|
||||
for (const candidate of extractScriptDocuments(source, entry.path)) {
|
||||
validateWorkspaceSource(candidate.source, candidate.label, {
|
||||
requireWorkspace: false,
|
||||
expandable: candidate.expandable,
|
||||
path: entry.path,
|
||||
rawBlock: candidate.rawBlock,
|
||||
});
|
||||
}
|
||||
}
|
||||
validatePythonRevisionStates(pythonPolicies);
|
||||
}
|
||||
|
||||
export function decodeManifest(bytes) {
|
||||
const fields = bytes.toString("utf8").split("\0");
|
||||
if (fields.at(-1) !== "") throw new Error("workspace verifier manifest is not NUL-terminated");
|
||||
fields.pop();
|
||||
if (fields.length % 2 !== 0) throw new Error("workspace verifier manifest has an incomplete record");
|
||||
const entries = [];
|
||||
for (let index = 0; index < fields.length; index += 2) {
|
||||
entries.push({ kind: fields[index], path: fields[index + 1] });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function cliArguments(argv) {
|
||||
let root;
|
||||
let manifest;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const option = argv[index];
|
||||
const value = argv[index + 1];
|
||||
if ((option === "--root" || option === "--manifest") && value !== undefined) {
|
||||
if (option === "--root" && root === undefined) root = value;
|
||||
else if (option === "--manifest" && manifest === undefined) manifest = value;
|
||||
else throw new Error(`duplicate or invalid option: ${option}`);
|
||||
index += 1;
|
||||
} else {
|
||||
throw new Error(`unknown or incomplete option: ${option}`);
|
||||
}
|
||||
}
|
||||
if (root === undefined || manifest === undefined) {
|
||||
throw new Error("usage: verify-workspace-descriptor-files.mjs --root ROOT --manifest NUL_FILE");
|
||||
}
|
||||
return { root, manifest };
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const { root, manifest } = cliArguments(argv);
|
||||
const manifestEntry = await lstat(manifest);
|
||||
if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink()) {
|
||||
throw new Error("workspace verifier manifest is not a regular file");
|
||||
}
|
||||
const entries = decodeManifest(await readFile(manifest));
|
||||
await verifyEntries({ root, entries });
|
||||
}
|
||||
|
||||
if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
|
||||
main(process.argv.slice(2)).catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -1,978 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
|
||||
import { extractScriptDocuments, verifyEntries } from "./verify-workspace-descriptor-files.mjs";
|
||||
|
||||
const repositoryRoot = fileURLToPath(new URL("../..", import.meta.url));
|
||||
const canonicalDescriptor = await readFile(join(repositoryRoot, "deploy/workspaces/example.yaml"), "utf8");
|
||||
|
||||
async function fixture(t) {
|
||||
const root = await mkdtemp(join(tmpdir(), "thoth-workspace-yaml-verifier-"));
|
||||
t.after(() => rm(root, { recursive: true, force: true }));
|
||||
return root;
|
||||
}
|
||||
|
||||
async function put(root, path, content) {
|
||||
await mkdir(dirname(join(root, path)), { recursive: true });
|
||||
await writeFile(join(root, path), content);
|
||||
}
|
||||
|
||||
function entry(kind, path) {
|
||||
return { kind, path };
|
||||
}
|
||||
|
||||
function bashN(root, path) {
|
||||
execFileSync("/bin/bash", ["-n", join(root, path)], { stdio: "pipe" });
|
||||
}
|
||||
|
||||
function replaceWorkspaceKeys(source, workspaceKey, schemaLine) {
|
||||
return source
|
||||
.replace(/^workspace:$/m, workspaceKey)
|
||||
.replace(/^ schema_version: 3$/m, schemaLine);
|
||||
}
|
||||
|
||||
test("production parser accepts semantic v3 with quoted Unicode/tagged keys and spacing", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const unicode = replaceWorkspaceKeys(
|
||||
canonicalDescriptor,
|
||||
'"\\u0077orkspace" :',
|
||||
' "\\u0073chema_version" : 3',
|
||||
);
|
||||
const tagged = replaceWorkspaceKeys(
|
||||
canonicalDescriptor,
|
||||
"!!str workspace :",
|
||||
" !!str schema_version : 3",
|
||||
);
|
||||
await put(root, "deploy/workspaces/unicode.yaml", unicode);
|
||||
await put(root, "deploy/workspaces/tagged.yaml", tagged);
|
||||
await verifyEntries({
|
||||
root,
|
||||
entries: [
|
||||
entry("workspace_descriptor", "deploy/workspaces/unicode.yaml"),
|
||||
entry("workspace_descriptor", "deploy/workspaces/tagged.yaml"),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("production parser rejects fancy keys with every non-v3 or ambiguous value", async (t) => {
|
||||
const invalid = [
|
||||
["unicode-v2", '"\\u0077orkspace" :', ' "\\u0073chema_version" : 2'],
|
||||
["tagged-leading-zero", "!!str workspace :", " !!str schema_version : 02"],
|
||||
["hexadecimal", "workspace :", " schema_version : 0x2"],
|
||||
["multiline", "workspace :", " schema_version : >\n 3"],
|
||||
["duplicate", "workspace :", " schema_version : 3\n schema_version: 3"],
|
||||
["inline", "workspace: { schema_version: 3 }", " schema_version: 3"],
|
||||
];
|
||||
for (const [name, workspaceKey, schemaLine] of invalid) {
|
||||
await t.test(name, async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), `thoth-workspace-yaml-${name}-`));
|
||||
try {
|
||||
const source = replaceWorkspaceKeys(canonicalDescriptor, workspaceKey, schemaLine);
|
||||
const path = `deploy/workspaces/${name}.yaml`;
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("Bash embedded workspace mappings are rejected while tracked-fixture-only bundles pass", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const validScript = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'WORKSPACE_YAML'",
|
||||
canonicalDescriptor.trimEnd(),
|
||||
"WORKSPACE_YAML",
|
||||
"cat <<'BUNDLE_YAML'",
|
||||
"bundle:",
|
||||
" name: deploy",
|
||||
"schema_version: 1",
|
||||
"job:",
|
||||
" state: operational",
|
||||
"BUNDLE_YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, "scripts/operator-smoke.sh", validScript);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator-smoke.sh")] }),
|
||||
/embedded workspace descriptor/i,
|
||||
);
|
||||
|
||||
const bundleScript = validScript.replace(canonicalDescriptor.trimEnd(), "job:\n name: deploy");
|
||||
await put(root, "scripts/operator-smoke.sh", bundleScript);
|
||||
await verifyEntries({
|
||||
root,
|
||||
entries: [entry("deployment_script", "scripts/operator-smoke.sh")],
|
||||
});
|
||||
});
|
||||
|
||||
test("PowerShell embedded workspace mappings are rejected while bundle-only strings pass", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const source = [
|
||||
"$workspace = @'",
|
||||
canonicalDescriptor.replace(" schema_version: 3", " schema_version: 0x2").trimEnd(),
|
||||
"'@",
|
||||
'$bundle = @"',
|
||||
"bundle:",
|
||||
" schema_version: 1",
|
||||
'"@',
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, "scripts/operator.ps1", source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator.ps1")] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("workspace descriptor family entries require a top-level workspace", async (t) => {
|
||||
const root = await fixture(t);
|
||||
await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n schema_version: 3\n");
|
||||
await assert.rejects(
|
||||
verifyEntries({
|
||||
root,
|
||||
entries: [entry("workspace_descriptor", "scripts/fixtures/workspace-registry-future.yaml")],
|
||||
}),
|
||||
/top-level workspace/i,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("script scalar workspace remains a bundle even with descriptor-like siblings", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/job-smoke.sh";
|
||||
const job = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'JOB-YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"schema_version: 2",
|
||||
"state: operational",
|
||||
"JOB-YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, job);
|
||||
bashN(root, path);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
|
||||
const bundles = [
|
||||
job.replace("job: refresh", "dwh:\n engine: postgres"),
|
||||
job.replace("job: refresh", "evidence:\n source: bundle"),
|
||||
];
|
||||
for (const bundle of bundles) {
|
||||
await put(root, path, bundle);
|
||||
bashN(root, path);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
test("standalone descriptor files require workspace to be a mapping", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/fixtures/workspace-registry-scalar.yaml";
|
||||
await put(root, path, "workspace: analytics\nschema_version: 3\n");
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }),
|
||||
/workspace.*mapping/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("Bash extractor supports hyphen, digit, escaped delimiters, and tab stripping", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
{
|
||||
name: "hyphen-v2",
|
||||
opener: "cat <<'WORKSPACE-YAML'",
|
||||
delimiter: "WORKSPACE-YAML",
|
||||
descriptor: canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2"),
|
||||
rejected: true,
|
||||
},
|
||||
{
|
||||
name: "digit-v3",
|
||||
opener: "cat <<2YAML",
|
||||
delimiter: "2YAML",
|
||||
descriptor: canonicalDescriptor,
|
||||
rejected: true,
|
||||
},
|
||||
{
|
||||
name: "escaped-v2",
|
||||
opener: "cat <<WORKSPACE\\-YAML",
|
||||
delimiter: "WORKSPACE-YAML",
|
||||
descriptor: canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2"),
|
||||
rejected: true,
|
||||
},
|
||||
{
|
||||
name: "tab-strip-v3",
|
||||
opener: "cat <<-'TAB-YAML'",
|
||||
delimiter: "\tTAB-YAML",
|
||||
descriptor: canonicalDescriptor.split("\n").map((line) => `\t${line}`).join("\n"),
|
||||
rejected: true,
|
||||
},
|
||||
];
|
||||
for (const item of cases) {
|
||||
await t.test(item.name, async () => {
|
||||
const path = `scripts/${item.name}-smoke.sh`;
|
||||
const source = ["#!/usr/bin/env bash", item.opener, item.descriptor.trimEnd(), item.delimiter, ""].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
const verification = verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
if (item.rejected) await assert.rejects(verification, /workspace descriptor/i);
|
||||
else await verification;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("unsupported Bash heredoc opener fails closed while a bundle heredoc stays allowed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const unsupportedPath = "scripts/unsupported-smoke.sh";
|
||||
const unsupported = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<$DELIMITER",
|
||||
canonicalDescriptor.trimEnd(),
|
||||
"$DELIMITER",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, unsupportedPath, unsupported);
|
||||
bashN(root, unsupportedPath);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", unsupportedPath)] }),
|
||||
/unsupported Bash heredoc opener/i,
|
||||
);
|
||||
|
||||
const bundlePath = "scripts/bundle-smoke.sh";
|
||||
const bundle = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'BUNDLE-YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"schema_version: 1",
|
||||
"state: operational",
|
||||
"BUNDLE-YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, bundlePath, bundle);
|
||||
bashN(root, bundlePath);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", bundlePath)] });
|
||||
});
|
||||
|
||||
|
||||
test("non-stripping heredoc close requires an exact physical delimiter line", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/trailing-close-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'---'",
|
||||
"--- ",
|
||||
canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(),
|
||||
"---",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("delimiter-like body lines remain content until a real exact close", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/delimiter-content-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'END'",
|
||||
"END ",
|
||||
" END",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"schema_version: 1",
|
||||
"END",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
const [candidate] = extractScriptDocuments(source, path);
|
||||
assert.match(candidate.source, /^END \n END\n/u);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("double-quoted non-special backslash is preserved in the delimiter", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/double-quoted-nonspecial-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
'cat <<"\\---"',
|
||||
"---",
|
||||
canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(),
|
||||
"\\---",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("double-quoted delimiter quote removal matches Bash special escapes", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["dollar", 'cat <<"DOL\\$LAR"', "DOL$LAR"],
|
||||
["backtick", 'cat <<"TIC\\`K"', "TIC`K"],
|
||||
["quote", 'cat <<"QUO\\\"TE"', 'QUO"TE'],
|
||||
["backslash", 'cat <<"SLA\\\\SH"', "SLA\\SH"],
|
||||
["newline", 'cat <<"LINE\\\nBREAK"', "LINEBREAK"],
|
||||
["nonspecial", 'cat <<"NON\\-SPECIAL"', "NON\\-SPECIAL"],
|
||||
];
|
||||
for (const [name, opener, close] of cases) {
|
||||
const path = `scripts/double-quoted-${name}-smoke.sh`;
|
||||
const source = ["#!/usr/bin/env bash", opener, "job: refresh", close, ""].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.equal(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), "job: refresh\n");
|
||||
assert.equal(extractScriptDocuments(source, path)[0].source, "job: refresh\n");
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("split heredoc operator continuation cannot bypass v2 validation", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/split-operator-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <\\",
|
||||
"<'YAML'",
|
||||
canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(),
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("multiple opener continuations are joined before heredoc discovery", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/multiple-continuation-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat \\",
|
||||
"<\\",
|
||||
"<'YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.equal(
|
||||
execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }),
|
||||
"job: refresh\nworkspace: analytics\n",
|
||||
);
|
||||
const [candidate] = extractScriptDocuments(source, path);
|
||||
assert.equal(candidate.label, `${path}:5 Bash heredoc`);
|
||||
assert.equal(candidate.source, "job: refresh\nworkspace: analytics\n");
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
test("backslash-newline inside single quotes is not removed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/single-quoted-noncontinuation-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"printf '%s' 'literal\\",
|
||||
"continued'",
|
||||
"cat <<'YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.equal(
|
||||
execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }),
|
||||
"literal\\\ncontinuedjob: refresh\nworkspace: analytics\n",
|
||||
);
|
||||
const [candidate] = extractScriptDocuments(source, path);
|
||||
assert.equal(candidate.label, `${path}:5 Bash heredoc`);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("PowerShell comment backslash cannot hide a following v2 here-string", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/powershell-comment-smoke.ps1";
|
||||
const source = [
|
||||
"# harmless PowerShell comment \\",
|
||||
"$workspace = @'",
|
||||
canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(),
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("PowerShell dialect accepts normal v3 and non-workspace bundle here-strings", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/powershell-valid-smoke.ps1";
|
||||
const source = [
|
||||
"$workspace = @'",
|
||||
canonicalDescriptor.trimEnd(),
|
||||
"'@",
|
||||
"$bundle = @'",
|
||||
"evidence:",
|
||||
" source: bundle",
|
||||
"schema_version: 2",
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/embedded workspace descriptor/i,
|
||||
);
|
||||
|
||||
const bundleOnly = [
|
||||
"$bundle = @'",
|
||||
"evidence:",
|
||||
" source: bundle",
|
||||
"schema_version: 2",
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, bundleOnly);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
test("unknown deployment script dialect fails closed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/operator-smoke.cmd";
|
||||
await put(root, path, "echo harmless\n");
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/unknown deployment script dialect/i,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("PowerShell cast and concatenation openers cannot hide embedded descriptors", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [name, opener] of [["cast", "[string]@'"], ["concat", "+@'"]]) {
|
||||
const path = `scripts/powershell-${name}-smoke.ps1`;
|
||||
const source = [opener, canonicalDescriptor.trimEnd(), "'@", ""].join("\n");
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/embedded workspace descriptor/i,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("expandable YAML interpolation that can hide a workspace descriptor fails closed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["braced-key", "${key}:\n schema_version: 3"],
|
||||
["plain-key", "$key:\n schema_version: 3"],
|
||||
["quoted-key", '"$key" :\n schema_version: 3'],
|
||||
["subexpression-key", "$($key):\n schema_version: 3"],
|
||||
["version", "workspace:\n schema_version: $version"],
|
||||
];
|
||||
for (const [name, body] of cases) {
|
||||
const path = `scripts/powershell-interpolation-${name}.ps1`;
|
||||
await put(root, path, [`$yaml = @\"`, body, `\"@`, ""].join("\n"));
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/interpolation|embedded workspace descriptor/i,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("Bash heredoc discovery ignores quoted, comment, here-string, and arithmetic tokens", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/bash-lexer-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
`printf '%s\\n' \"cat <<'QUOTED'\"`,
|
||||
`printf '%s\\n' 'cat <<\"SINGLE\"'`,
|
||||
"# cat <<'COMMENT'",
|
||||
"value=$((1 << 2))",
|
||||
`cat <<< \"not a heredoc\"`,
|
||||
"cat <<'YAML'",
|
||||
"job: refresh",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
const extracted = extractScriptDocuments(source, path);
|
||||
assert.equal(extracted.length, 1);
|
||||
assert.equal(extracted[0].source, "job: refresh\n");
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
test("UTF-8 decoding is fatal but literal replacement characters are valid text", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const validPath = "deploy/workspaces/replacement.yaml";
|
||||
await put(root, validPath, `${canonicalDescriptor}# literal replacement: �\n`);
|
||||
await verifyEntries({ root, entries: [entry("workspace_descriptor", validPath)] });
|
||||
|
||||
const invalidPath = "deploy/workspaces/malformed.yaml";
|
||||
await mkdir(dirname(join(root, invalidPath)), { recursive: true });
|
||||
await writeFile(join(root, invalidPath), Buffer.concat([Buffer.from(canonicalDescriptor), Buffer.from([0xff])]));
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("workspace_descriptor", invalidPath)] }),
|
||||
/valid UTF-8/i,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("unmarked expandable Bash YAML cannot generate descriptor keys or values at runtime", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["quoted", '"$key" :'],
|
||||
["command", "$(printf workspace):"],
|
||||
["braced", "${key}:"],
|
||||
["plain", "$key:"],
|
||||
];
|
||||
for (const [name, generatedKey] of cases) {
|
||||
const path = `scripts/bash-dynamic-${name}.sh`;
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"key=workspace",
|
||||
"cat <<YAML",
|
||||
generatedKey,
|
||||
" schema_version: 3",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /workspace/u);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/i,
|
||||
);
|
||||
}
|
||||
|
||||
const valuePath = "scripts/bash-dynamic-value.sh";
|
||||
const valueSource = [
|
||||
"#!/usr/bin/env bash",
|
||||
"version=3",
|
||||
"cat <<YAML",
|
||||
"workspace:",
|
||||
" schema_version: $version",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, valuePath, valueSource);
|
||||
bashN(root, valuePath);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", valuePath)] }),
|
||||
/exact-content reviewed allowlist/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("an in-band marker cannot authorize expandable content", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [path, source] of [
|
||||
["scripts/fake-marker.sh", [
|
||||
"#!/usr/bin/env bash",
|
||||
"# schema-v3-only: expandable-nonworkspace",
|
||||
"cat <<YAML",
|
||||
"${DESCRIPTOR}",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n")],
|
||||
["scripts/fake-marker.ps1", [
|
||||
"# schema-v3-only: expandable-nonworkspace",
|
||||
'$yaml = @"',
|
||||
"$descriptor",
|
||||
'"@',
|
||||
"",
|
||||
].join("\n")],
|
||||
]) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("current exact reviewed expandable blocks pass only at their trusted paths", async (t) => {
|
||||
const reviewedPaths = [
|
||||
"scripts/test-server-pi-state-topology.sh",
|
||||
"scripts/test-vector-backup-restore-safety.sh",
|
||||
"scripts/test-windows-clone-contract.ps1",
|
||||
"scripts/unified-deployment-smoke.sh",
|
||||
"scripts/vector-backup.sh",
|
||||
"scripts/vector-restore.sh",
|
||||
];
|
||||
await verifyEntries({
|
||||
root: repositoryRoot,
|
||||
entries: reviewedPaths.map((path) => entry("deployment_script", path)),
|
||||
});
|
||||
});
|
||||
|
||||
test("PowerShell tokenizer ignores opener text in comments and ordinary strings", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/powershell-lexical-context.ps1";
|
||||
const source = [
|
||||
"# example @'",
|
||||
'\"example @\'\"',
|
||||
"'example @\"'",
|
||||
"<# block @'",
|
||||
"still @\" #>",
|
||||
"$cast = [string]@'",
|
||||
"job: cast",
|
||||
"'@",
|
||||
"$concat = $cast +@'",
|
||||
"job: concat",
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
const extracted = extractScriptDocuments(source, path);
|
||||
assert.equal(extracted.length, 2);
|
||||
assert.deepEqual(extracted.map((item) => item.source), ["job: cast\n", "job: concat\n"]);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("policy text rejects NUL and prescribed symbol substrings but permits lower-camel legacy identifiers", async (t) => {
|
||||
const root = await fixture(t);
|
||||
await put(root, "backend/src/nul.ts", Buffer.from("safe\0WorkspaceV2"));
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", "backend/src/nul.ts")] }), /NUL byte/);
|
||||
|
||||
for (const [name, text] of [
|
||||
["compat", "type X = WorkspaceV2Compat;"],
|
||||
["mixed-prescribed", "type X = wOrKsPaCeV2;"],
|
||||
["lower-deprecated", "type X = deprecatedV2Descriptor;"],
|
||||
["upper-function", "WRITEMIGRATEDWORKSPACE(value);"],
|
||||
["adapter", "type X = LegacyWorkspaceAdapter;"],
|
||||
["lower", "type X = legacyworkspace;"],
|
||||
["mixed", "type X = LeGaCyWoRkSpAcE;"],
|
||||
]) {
|
||||
const path = `backend/src/${name}.ts`;
|
||||
await put(root, path, text);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /forbidden/);
|
||||
}
|
||||
await put(root, "backend/src/allowed.ts", "const legacyWorkspacePath = value;");
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "backend/src/allowed.ts")] });
|
||||
});
|
||||
|
||||
test("revision-state structural scan permits only the exact historical decoder occurrence", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
await put(root, registry, 'if (revision.state !== "operational") return;\n');
|
||||
await verifyEntries({ root, entries: [entry("policy_text", registry)] });
|
||||
|
||||
const variants = [
|
||||
'if (revision.state !== "operational") return;\nif (revision["state"] === value) return;\n',
|
||||
'if (workspaceRevision\n .state === value) return;\n',
|
||||
"if (selectedWorkspace [ 'state' ] === value) return;\n",
|
||||
];
|
||||
for (let index = 0; index < variants.length; index += 1) {
|
||||
const path = index === 0 ? registry : `frontend/src/revision-${index}.ts`;
|
||||
await put(root, path, variants[index]);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("complete descriptors supplied only through Bash or PowerShell variables require exact review", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["scripts/variable-descriptor.sh", ["#!/usr/bin/env bash", "cat <<YAML", "${DESCRIPTOR}", "YAML", ""].join("\n")],
|
||||
["scripts/variable-descriptor.ps1", ['$yaml = @"', "$descriptor", '"@', ""].join("\n")],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("all Bash and PowerShell positional or special dollar expansions fail without exact review", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["scripts/positional.sh", "cat <<YAML\n$1\nYAML\n"],
|
||||
["scripts/all-args.sh", "cat <<YAML\n$@\nYAML\n"],
|
||||
["scripts/positional.ps1", '$yaml = @"\n$1\n"@\n'],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("PowerShell backtick escapes hash and quote tokens without hiding a later real here-string", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [name, prefix] of [
|
||||
["escaped-hash", "Write-Output `# harmless"],
|
||||
["escaped-quote", 'Write-Output `" harmless'],
|
||||
]) {
|
||||
const path = `scripts/${name}.ps1`;
|
||||
const source = [prefix, "$yaml = @'", "workspace:", " schema_version: 2", "'@", ""].join("\n");
|
||||
await put(root, path, source);
|
||||
assert.equal(extractScriptDocuments(source, path).length, 1);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/embedded workspace descriptor/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("TypeScript AST rejects comment-separated and destructured revision state", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [index, source] of [
|
||||
"const value = revision /*legacy*/ . state;",
|
||||
"const { state } = revision;",
|
||||
"const { state: oldState } = selectedWorkspace;",
|
||||
].entries()) {
|
||||
const path = `frontend/src/ast-revision-${index}.ts`;
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
}
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
await put(root, registry, 'if (revision.state !== "operational") return;\nconst { state } = revision;\n');
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/);
|
||||
await put(root, "backend/src/unrelated.ts", "const { state } = lease; const jobState = job.state;");
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "backend/src/unrelated.ts")] });
|
||||
});
|
||||
|
||||
|
||||
test("AST recognizes semantic state keys in every revision destructuring form", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["backend/src/computed.mts", 'const { ["state"]: oldState } = revision;'],
|
||||
["frontend/src/renamed.cts", 'const { "state": oldState = fallback } = workspaceRevision;'],
|
||||
["backend/scripts/template.TS", 'const { [`state`]: oldState } = selectedWorkspace;'],
|
||||
["scripts/parameter.txt", 'function read({ state: oldState = fallback } = revision) {}'],
|
||||
["scripts/assignment.sh", '({ state } = workspaceRevision);'],
|
||||
["scripts/computed-assignment.data", '({ ["state"]: oldState = fallback } = selectedWorkspace);'],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("policy_text", path)] }),
|
||||
/revision-state/,
|
||||
path,
|
||||
);
|
||||
}
|
||||
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
await put(root, registry, [
|
||||
'if (revision.state !== "operational") return;',
|
||||
'function read({ ["state"]: oldState } = revision) {}',
|
||||
"",
|
||||
].join("\n"));
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("policy_text", registry)] }),
|
||||
/revision-state/,
|
||||
);
|
||||
});
|
||||
|
||||
test("tolerant all-suffix AST scan ignores strings/comments and unrelated state", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/arbitrary.weird";
|
||||
await put(root, path, [
|
||||
'// const { state } = revision;',
|
||||
'"revision.state";',
|
||||
"'({ [\\\"state\\\"]: oldState } = selectedWorkspace)';",
|
||||
"const { state } = lease;",
|
||||
"const jobState = job.state;",
|
||||
"record.state = 'ready';",
|
||||
"",
|
||||
].join("\n"));
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("computed revision destructuring keys fold parentheses assertions templates and string concatenation", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["backend/src/paren.ts", 'const { [("state")]: oldState } = revision;'],
|
||||
["backend/src/concat.ts", 'const { ["st" + "ate"]: oldState } = workspaceRevision;'],
|
||||
["frontend/src/template.ts", 'const { [`st${"ate"}`]: oldState } = selectedWorkspace;'],
|
||||
["scripts/assertion.data", 'const { [("st" as string) + (`ate` satisfies string)]: oldState } = revision;'],
|
||||
["scripts/assignment.txt", '({ ["st" + "ate"]: oldState } = selectedWorkspace);'],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
for (const injected of [
|
||||
'const { [("state")]: oldState } = revision;',
|
||||
'({ ["st" + "ate"]: oldState } = revision);',
|
||||
]) {
|
||||
await put(root, registry, `if (revision.state !== "operational") return;\n${injected}\n`);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/);
|
||||
}
|
||||
});
|
||||
|
||||
test("polyglot masking and JSX syntax prevent comment and string false positives", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const passing = [
|
||||
["backend/scripts/comment.py", '# revision.state\nvalue = "revision.state"\ntext = """selectedWorkspace.state"""\n'],
|
||||
["scripts/comment.ps1", '# revision.state\n<# workspaceRevision.state #>\n$value = "revision.state"\n'],
|
||||
["scripts/comment.sh", '# revision.state\nprintf \'%s\\n\' "selectedWorkspace.state"\n'],
|
||||
["frontend/src/content.tsx", 'export const view = <div>revision.state</div>;'],
|
||||
["frontend/src/attribute.tsx", 'export const view = <div title="revision.state" />;'],
|
||||
["frontend/src/expression.tsx", 'export const view = <div>{"revision.state"}</div>;'],
|
||||
["scripts/arbitrary.data", 'title: "revision.state"\n# const { state } = revision\nlease:\n state: ready\n'],
|
||||
];
|
||||
for (const [path, source] of passing) {
|
||||
await put(root, path, source);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
|
||||
for (const [path, source] of [
|
||||
["scripts/code.txt", "const { state } = revision;"],
|
||||
["scripts/code.data", '({ ["st" + "ate"]: oldState } = workspaceRevision);'],
|
||||
]) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("rest bindings and dynamic computed keys are not semantic state-property access", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["backend/src/rest.ts", "const { ...state } = revision;"],
|
||||
["frontend/src/renamed.ts", "const { other: state } = workspaceRevision;"],
|
||||
["scripts/dynamic.txt", "const { [state]: value } = selectedWorkspace;"],
|
||||
["scripts/dynamic-assignment.data", "({ [state]: value } = revision);"],
|
||||
["scripts/spread-assignment.data", "({ ...state } = workspaceRevision);"],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
test("polyglot code remains structural across shell Python PowerShell YAML TSX and JSX", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["scripts/code.sh", "value=revision.state\n"],
|
||||
["scripts/code.ps1", "$value = workspaceRevision.state\n"],
|
||||
["backend/scripts/code.py", "value = selectedWorkspace.state\n"],
|
||||
["scripts/code.yaml", "value: revision.state\n"],
|
||||
["frontend/src/code.tsx", "export const view = <div>{revision.state}</div>;"],
|
||||
["frontend/src/code.jsx", "export const view = <div>{workspaceRevision.state}</div>;"],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("PowerShell executable subexpressions expose dollar-prefixed revision access", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["scripts/ps-property.ps1", 'Write-Output "revision: $($revision.state)"\n'],
|
||||
["scripts/ps-element.ps1", 'Write-Output "$($workspaceRevision[\'state\'])"\n'],
|
||||
["scripts/ps-workspace.ps1", '$value = $workspaceRevision.state\n'],
|
||||
["scripts/ps-nested.ps1", 'Write-Output "$($($revision.state))"\n'],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
const passing = [
|
||||
'# $revision.state\nWrite-Output "revision.state"\n',
|
||||
"Write-Output '$selectedWorkspace[\"state\"]'\n",
|
||||
];
|
||||
for (let index = 0; index < passing.length; index += 1) {
|
||||
const path = `scripts/ps-literal-${index}.ps1`;
|
||||
await put(root, path, passing[index]);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
test("Python f-string fields expose revision access while literal text remains masked", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["backend/scripts/f-property.py", 'value = f"{revision.state}"\n'],
|
||||
["backend/scripts/fr-element.py", 'value = fr"{workspaceRevision[\'state\']}"\n'],
|
||||
["backend/scripts/rf-element.py", 'value = rf"prefix {selectedWorkspace[\"state\"]}"\n'],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
const passing = [
|
||||
'value = f"revision.state"\n',
|
||||
'value = f"{{revision.state}}"\n',
|
||||
'value = "revision.state"\n',
|
||||
'value = r"workspaceRevision.state"\n',
|
||||
'value = """selectedWorkspace.state"""\n',
|
||||
'value = r"""revision.state"""\n',
|
||||
];
|
||||
for (let index = 0; index < passing.length; index += 1) {
|
||||
const path = `backend/scripts/python-literal-${index}.py`;
|
||||
await put(root, path, passing[index]);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("Bash masking preserves parameter trimming and executable command consumers", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["scripts/trim.sh", "trimmed=${value#prefix}; old=revision.state\n"],
|
||||
["scripts/base.sh", "base=${path##*/}; old=workspaceRevision.state\n"],
|
||||
["scripts/backtick.sh", "old=`echo revision.state`\n"],
|
||||
["scripts/quoted-backtick.sh", 'echo "old: `echo revision.state`"\n'],
|
||||
["scripts/jq.sh", "jq '.revision.state' snapshot.json\n"],
|
||||
["scripts/substitution.sh", 'echo "$(echo revision.state)"\n'],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
await put(root, "scripts/echo.sh", 'echo "revision.state"\n# workspaceRevision.state\n');
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "scripts/echo.sh")] });
|
||||
await put(root, "scripts/literal.yaml", '# revision.state\nvalue: "selectedWorkspace.state"\n');
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "scripts/literal.yaml")] });
|
||||
});
|
||||
|
||||
test("YAML keeps URL slashes as data rather than a false line comment", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/url.yaml";
|
||||
await put(root, path, "url: https://host/x; old: selectedWorkspace.state\n");
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
});
|
||||
@@ -1,494 +0,0 @@
|
||||
import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify";
|
||||
import cors from "@fastify/cors";
|
||||
import cookie from "@fastify/cookie";
|
||||
import rateLimit from "@fastify/rate-limit";
|
||||
import { dirname, isAbsolute, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
import { PiProcessManager } from "./pi/pi-process-manager.js";
|
||||
import { SseHub } from "./sse/sse-hub.js";
|
||||
import { authenticateSession, captureAuthConfigSnapshot, configuredOrigin } from "./auth/auth.js";
|
||||
import type { PrincipalContext } from "./auth/principal.js";
|
||||
import type { LoadedAuthConfig } from "./auth/types.js";
|
||||
import { createCurrentLocalUserRegistryResolver, type LocalUserRegistry } from "./auth/local-registry.js";
|
||||
import { AuthSessionOperationalError, createFileAuthSessionStore, type AuthSessionStore, type AuthSessionValidity } from "./auth/session-store.js";
|
||||
import type { WindowsAuthStorageBridge } from "./auth/windows-auth-storage.js";
|
||||
import { registerAuthRoutes } from "./auth/routes.js";
|
||||
import { createOidcProtocol, type OidcProtocol, type OidcProtocolOptions } from "./auth/oidc-client.js";
|
||||
import { createConfiguredAuthDiagnoser } from "./auth/diagnostic-command.js";
|
||||
import type { AuthDiagnoser } from "./auth/diagnostics.js";
|
||||
import { isUsableAuthenticationSecret } from "./auth/secret-policy.js";
|
||||
import { secretValue } from "./config/secret-bundle.js";
|
||||
import { sessionRoutes } from "./routes/sessions.js";
|
||||
import { sqlRoutes } from "./routes/sql.js";
|
||||
import { metaRoutes, type ListModelsFn } from "./routes/meta.js";
|
||||
import { settingsRoutes, effectiveSettings } from "./routes/settings.js";
|
||||
import { createPiModelLister } from "./pi/list-models.js";
|
||||
import { createPiManagement, type PiManagementService } from "./pi/management.js";
|
||||
import { loadSettings, type Settings } from "./settings/settings-store.js";
|
||||
import { ReadinessManager } from "./runtime/readiness-manager.js";
|
||||
import { MaintenanceBarrier } from "./runtime/maintenance-gate.js";
|
||||
import { WorkspaceRegistry } from "./workspaces/registry.js";
|
||||
import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js";
|
||||
import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js";
|
||||
import { piManagementRoutes } from "./routes/pi-management.js";
|
||||
import { supportsSessionRuntime } from "./workspaces/bindings.js";
|
||||
import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js";
|
||||
import type { WorkspaceDescriptor } from "./workspaces/schema.js";
|
||||
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
||||
import { createCatalogRepository } from "./catalog/repository.js";
|
||||
import type { CatalogRepository } from "./catalog/types.js";
|
||||
import { CatalogService } from "./catalog/service.js";
|
||||
import { catalogDatabaseRoutes } from "./routes/catalog-databases.js";
|
||||
import { CatalogOperationCoordinator } from "./catalog/operation-coordinator.js";
|
||||
import { ConcreteCatalogPostgresAccess, type CatalogPostgresAccess } from "./catalog/postgres-access.js";
|
||||
import { CatalogTableService } from "./catalog/table-service.js";
|
||||
import { catalogTableRoutes } from "./routes/catalog-tables.js";
|
||||
import { ConcreteCatalogSchemaIntrospector, type CatalogSchemaIntrospector } from "./catalog/schema-introspector.js";
|
||||
import { CatalogSyncWorker } from "./catalog/sync-worker.js";
|
||||
import { catalogSchemaRoutes } from "./routes/catalog-schema.js";
|
||||
import {
|
||||
loadMetadataGenerationModels,
|
||||
type MetadataGenerationModels,
|
||||
} from "./catalog/metadata-generation-models.js";
|
||||
import { metadataGenerationModelRoutes } from "./routes/metadata-generation-models.js";
|
||||
import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-description-consolidation.js";
|
||||
import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js";
|
||||
import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js";
|
||||
import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js";
|
||||
import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js";
|
||||
import {
|
||||
ConcreteDescriptionSourceSampler,
|
||||
type DescriptionSourceSampler,
|
||||
} from "./catalog/description-source-sampler.js";
|
||||
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
|
||||
import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js";
|
||||
import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
|
||||
import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
mgr?: PiProcessManager;
|
||||
spawnFn?: () => any;
|
||||
listModels?: ListModelsFn;
|
||||
getSettings?: (principal?: PrincipalContext) => Settings | Promise<Settings>;
|
||||
readiness?: ReadinessManager;
|
||||
hub?: SseHub;
|
||||
workspaceRegistry?: WorkspaceRegistry;
|
||||
workspaceDiagnoser?: WorkspaceDiagnoser;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
catalogRepository?: CatalogRepository;
|
||||
catalogService?: CatalogService;
|
||||
catalogPostgresAccess?: CatalogPostgresAccess;
|
||||
catalogTableService?: CatalogTableService;
|
||||
catalogLogicalRelationshipService?: CatalogLogicalRelationshipService;
|
||||
effectiveRelationshipSnapshotProvider?: EffectiveRelationshipSnapshotProvider;
|
||||
catalogSchemaIntrospector?: CatalogSchemaIntrospector;
|
||||
catalogSyncWorker?: CatalogSyncWorker;
|
||||
catalogOperationCoordinator?: CatalogOperationCoordinator;
|
||||
metadataGenerationModels?: MetadataGenerationModels;
|
||||
modelCompleter?: ModelCompleter;
|
||||
descriptionSourceSampler?: DescriptionSourceSampler;
|
||||
workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
|
||||
maintenanceBarrier?: MaintenanceBarrier;
|
||||
piManagement?: PiManagementService;
|
||||
localUserRegistry?: LocalUserRegistry;
|
||||
authSessionStore?: AuthSessionStore;
|
||||
/** Explicit test-only transport seam; production always invokes the hidden tht bridge. */
|
||||
authStorageBridgeForTest?: WindowsAuthStorageBridge;
|
||||
oidcProtocol?: OidcProtocol;
|
||||
authDiagnoser?: AuthDiagnoser;
|
||||
/** Explicit test seam; production uses the provider-neutral OIDC constructor. */
|
||||
oidcProtocolFactory?: (options: OidcProtocolOptions) => OidcProtocol;
|
||||
}
|
||||
|
||||
export interface AppWithAuthSessionStore extends FastifyInstance {
|
||||
thothiiAuthSessionStore?: AuthSessionStore;
|
||||
}
|
||||
|
||||
export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstance {
|
||||
const app = Fastify({ logger: { level: "warn" }, disableRequestLogging: true });
|
||||
app.decorateRequest("authConfigSnapshot", undefined);
|
||||
app.decorateRequest("authConfigSnapshotCaptured", false);
|
||||
app.decorateRequest("authConfigSnapshotUnavailable", false);
|
||||
const isolatedTestRoot = process.env.VITEST === "true"
|
||||
? join(tmpdir(), `thothii-workspace-secrets-vitest-${process.pid}`)
|
||||
: undefined;
|
||||
const workspaceSecretStore = deps?.workspaceSecretStore ?? new WorkspaceSecretStore({
|
||||
root: isolatedTestRoot ?? config.workspaceSecretStoreRoot,
|
||||
runtimeRoot: isolatedTestRoot === undefined
|
||||
? config.workspaceSecretRuntimeRoot
|
||||
: join(isolatedTestRoot, "runtime"),
|
||||
installationId: config.workspaceRegistry.installationId,
|
||||
});
|
||||
|
||||
const cookieAuth = config.authMode === "local" || config.authMode === "oidc";
|
||||
app.register(cors, {
|
||||
// The delegator runs at CORS's onRequest hook. It owns the one request-scoped config load
|
||||
// which subsequent auth hooks and routes consume, including preflights that end here.
|
||||
delegator: (request, callback) => {
|
||||
const snapshot = captureAuthConfigSnapshot(request, config.authentication);
|
||||
const origin = configuredOrigin(snapshot);
|
||||
const snapshotUsesCookies = snapshot?.value.mode === "local" || snapshot?.value.mode === "oidc";
|
||||
callback(null, {
|
||||
origin: snapshotUsesCookies && origin ? corsOrigin(request, origin) : cookieAuth ? false : true,
|
||||
credentials: snapshotUsesCookies,
|
||||
methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
|
||||
});
|
||||
},
|
||||
});
|
||||
// Cookie parsing and the rate-limit plugin must precede every auth/application route.
|
||||
app.register(cookie);
|
||||
app.register(rateLimit, { global: false });
|
||||
|
||||
const tht = deps?.thtRunner ?? new ThtRunner({
|
||||
thtBin: config.thtBin,
|
||||
harnessDir: config.harnessDir,
|
||||
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
|
||||
dataRoot: config.dataRoot,
|
||||
runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"),
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
secretsFile: config.secretsFile,
|
||||
secretFiles: config.secretFiles,
|
||||
workspaceSecretStore,
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
});
|
||||
const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
|
||||
const hub = deps?.hub ?? new SseHub();
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
|
||||
const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
|
||||
installationFile: config.installationConfigFile,
|
||||
secretsFile: config.secretsFile,
|
||||
});
|
||||
const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
|
||||
pythonExecutable: isAbsolute(config.thtBin) ? join(dirname(config.thtBin), "python") : "python3",
|
||||
cwd: config.harnessDir,
|
||||
});
|
||||
const catalogPostgresAccess = deps?.catalogPostgresAccess ?? new ConcreteCatalogPostgresAccess(
|
||||
workspaceSecretStore,
|
||||
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
|
||||
);
|
||||
const descriptionSourceSampler = deps?.descriptionSourceSampler
|
||||
?? new ConcreteDescriptionSourceSampler(catalogPostgresAccess, workspaceSecretStore);
|
||||
const descriptionGenerationWorker = new DescriptionGenerationWorker(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
metadataGenerationModels,
|
||||
modelCompleter,
|
||||
catalogOperationCoordinator,
|
||||
descriptionSourceSampler,
|
||||
);
|
||||
const sensitiveDataSuggester = new SensitiveDataSuggester(
|
||||
catalogRepository,
|
||||
metadataGenerationModels,
|
||||
modelCompleter,
|
||||
);
|
||||
const sensitiveDataSuggestionRunner = new SensitiveDataSuggestionRunner(
|
||||
catalogRepository,
|
||||
sensitiveDataSuggester,
|
||||
);
|
||||
const catalogService = deps?.catalogService ?? new CatalogService(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
workspaceSecretStore,
|
||||
config.workspaceRegistry.secretRoots,
|
||||
config.workspaceDiagnosticTimeoutMs,
|
||||
catalogPostgresAccess,
|
||||
catalogOperationCoordinator,
|
||||
);
|
||||
const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
|
||||
const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
|
||||
?? new CatalogLogicalRelationshipService(catalogRepository);
|
||||
const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider
|
||||
?? (config.catalogDatabase === undefined
|
||||
? undefined
|
||||
: new EffectiveRelationshipSnapshotProvider(
|
||||
catalogRepository,
|
||||
catalogLogicalRelationshipService,
|
||||
catalogOperationCoordinator,
|
||||
));
|
||||
const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
|
||||
catalogPostgresAccess,
|
||||
workspaceSecretStore,
|
||||
);
|
||||
const catalogSyncWorker = deps?.catalogSyncWorker ?? new CatalogSyncWorker(
|
||||
catalogRepository,
|
||||
catalogSchemaIntrospector,
|
||||
catalogOperationCoordinator,
|
||||
config.catalogSyncTimeoutMs,
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await sensitiveDataSuggestionRunner.initialize(); });
|
||||
if (!deps?.catalogRepository && catalogRepository.close) {
|
||||
app.addHook("onClose", async () => { await catalogRepository.close?.(); });
|
||||
}
|
||||
app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
|
||||
app.addHook("onClose", async () => { await descriptionGenerationWorker.stop(); });
|
||||
const workspaceDiagnoser = deps?.workspaceDiagnoser
|
||||
?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
});
|
||||
const workspaceRuntimeSupport = deps?.workspaceRuntimeSupport ?? ((workspace: WorkspaceDescriptor) => {
|
||||
const lease = resolveRuntimeBindingsWithWorkspaceSecrets(
|
||||
workspace,
|
||||
process.env,
|
||||
config.workspaceRegistry.secretRoots,
|
||||
workspaceSecretStore,
|
||||
);
|
||||
try {
|
||||
return supportsSessionRuntime(lease.bindings);
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
});
|
||||
const readiness = deps?.readiness ?? new ReadinessManager(
|
||||
tht as ThtRunner,
|
||||
Math.round(config.ollamaEnsureTimeoutMs / 1000),
|
||||
);
|
||||
|
||||
const listModels = deps?.listModels ?? createPiModelLister(config, {
|
||||
warn: (detail) => app.log.warn(
|
||||
{ component: "pi-model-list", detail },
|
||||
"Pi enabled-model configuration warning",
|
||||
),
|
||||
});
|
||||
const runnerFor = (principal: PrincipalContext): any => {
|
||||
const candidate = tht as any;
|
||||
return typeof candidate.withPrincipal === "function" ? candidate.withPrincipal(principal) : candidate;
|
||||
};
|
||||
const getSettings = async (principal: PrincipalContext): Promise<Settings> => {
|
||||
if (deps?.getSettings) return await deps.getSettings(principal);
|
||||
const stored = loadSettings(config);
|
||||
const effective = effectiveSettings(config, stored);
|
||||
// In the registry system the legacy `harness/workspaces/*.yaml` default is obsolete: when no
|
||||
// installation workspace is pinned, default to the first active registry workspace.
|
||||
if (!stored.workspace) {
|
||||
try {
|
||||
const revisions = await workspaceRegistry.list();
|
||||
if (revisions.length > 0) effective.workspace = revisions[0].id;
|
||||
} catch {
|
||||
// Registry not bootstrapped yet; keep the legacy fallback.
|
||||
}
|
||||
}
|
||||
return effective;
|
||||
};
|
||||
const piManagement = deps?.piManagement ?? createPiManagement(config, { listModels });
|
||||
|
||||
const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile);
|
||||
const localRegistryResolver = deps?.localUserRegistry === undefined
|
||||
? createCurrentLocalUserRegistryResolver()
|
||||
: undefined;
|
||||
const resolveLocalUserRegistry = (loaded: LoadedAuthConfig) => {
|
||||
return deps?.localUserRegistry ?? localRegistryResolver?.resolve(loaded);
|
||||
};
|
||||
const localUserForSnapshot = async (loaded: LoadedAuthConfig, subject: string) => {
|
||||
try {
|
||||
if (loaded.value.mode !== "local") return { revision: loaded.revision, user: undefined };
|
||||
const registry = resolveLocalUserRegistry(loaded);
|
||||
if (!registry) throw new AuthSessionOperationalError();
|
||||
const user = await registry.findBySubject(subject);
|
||||
return {
|
||||
revision: loaded.revision,
|
||||
user: user === undefined ? undefined : {
|
||||
enabled: user.enabled,
|
||||
authRevision: user.authRevision,
|
||||
roles: user.roles,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof AuthSessionOperationalError) throw error;
|
||||
throw new AuthSessionOperationalError();
|
||||
}
|
||||
};
|
||||
const sessionValidityForSnapshot = (loaded: LoadedAuthConfig): AuthSessionValidity => ({
|
||||
currentAuthConfigRevision: () => loaded.revision,
|
||||
currentLocalUser: (subject) => localUserForSnapshot(loaded, subject),
|
||||
});
|
||||
const resolveOidcProtocol = (loaded: LoadedAuthConfig): OidcProtocol | undefined => {
|
||||
if (deps?.oidcProtocol) return deps.oidcProtocol;
|
||||
if (loaded.value.mode !== "oidc") return undefined;
|
||||
try {
|
||||
const clientSecret = secretValue(config, loaded.value.oidc.clientSecretRef);
|
||||
if (!isUsableAuthenticationSecret("THT_OIDC_CLIENT_SECRET", clientSecret)) return undefined;
|
||||
return (deps?.oidcProtocolFactory ?? createOidcProtocol)({
|
||||
issuer: loaded.value.oidc.issuer,
|
||||
clientId: loaded.value.oidc.clientId,
|
||||
clientSecret,
|
||||
callbackUrl: new URL("/api/auth/oidc/callback", loaded.value.publicUrl).href,
|
||||
scopes: loaded.value.oidc.scopes,
|
||||
groupsClaim: loaded.value.oidc.groupsClaim,
|
||||
});
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
const authDiagnoser = deps?.authDiagnoser ?? createConfiguredAuthDiagnoser(config, {
|
||||
localUserRegistry: resolveLocalUserRegistry,
|
||||
oidcProtocol: resolveOidcProtocol,
|
||||
});
|
||||
const authSessionStore = deps?.authSessionStore ?? (config.authMode === "local" || config.authMode === "oidc"
|
||||
? createFileAuthSessionStore(config.authStateRoot, {
|
||||
currentAuthConfigRevision: () => {
|
||||
try {
|
||||
return config.authentication?.current().revision ?? "";
|
||||
} catch {
|
||||
throw new AuthSessionOperationalError();
|
||||
}
|
||||
},
|
||||
currentLocalUser: async (subject) => {
|
||||
try {
|
||||
const loaded = config.authentication?.current();
|
||||
if (!loaded) return { revision: "", user: undefined };
|
||||
return await localUserForSnapshot(loaded, subject);
|
||||
} catch (error) {
|
||||
if (error instanceof AuthSessionOperationalError) throw error;
|
||||
throw new AuthSessionOperationalError();
|
||||
}
|
||||
},
|
||||
}, deps?.authStorageBridgeForTest === undefined
|
||||
? undefined
|
||||
: process.platform === "win32"
|
||||
? { windowsStorageBridge: deps.authStorageBridgeForTest }
|
||||
: { posixStorageBridge: deps.authStorageBridgeForTest })
|
||||
: undefined);
|
||||
(app as AppWithAuthSessionStore).thothiiAuthSessionStore = authSessionStore;
|
||||
const authenticate = authenticateSession({
|
||||
mode: config.authMode,
|
||||
publicExposure: config.publicExposure,
|
||||
authentication: config.authentication,
|
||||
sessionStore: authSessionStore,
|
||||
sessionValidityForSnapshot,
|
||||
});
|
||||
app.addHook("preHandler", (req, reply, done) => {
|
||||
if (isMaintenanceControl(req.url)) {
|
||||
if (!isLoopback(req.ip)) {
|
||||
reply.code(403).send({ error: "loopback maintenance control required" });
|
||||
}
|
||||
}
|
||||
done();
|
||||
});
|
||||
app.addHook("preHandler", authenticate);
|
||||
app.get("/health", async () => ({ status: "ok" }));
|
||||
app.get("/health/dwh", async () => {
|
||||
// In the registry system there is no single legacy DWH config: ping the first active
|
||||
// workspace's rendered runtime config. If the registry is not bootstrapped yet, do not
|
||||
// block the app — per-workspace diagnostics and the session precheck own reachability.
|
||||
try {
|
||||
const revisions = await workspaceRegistry.list();
|
||||
if (revisions.length > 0) {
|
||||
return await tht.dbPing(revisions[0].snapshotPath);
|
||||
}
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
return { ok: true, detail: "workspace diagnostics own DWH reachability" };
|
||||
});
|
||||
registerAuthRoutes(app, {
|
||||
authMode: config.authMode,
|
||||
authentication: config.authentication,
|
||||
sessionStore: authSessionStore,
|
||||
localUserRegistry: deps?.localUserRegistry,
|
||||
resolveLocalUserRegistry,
|
||||
resolveOidcProtocol,
|
||||
});
|
||||
sessionRoutes(app, {
|
||||
mgr, tht: tht as ThtRunner, hub, getSettings, readiness, listModels, workspaceRegistry,
|
||||
dwhPrecheck: config.dwhPrecheck,
|
||||
legacyWorkspaceMode: config.legacyWorkspaceMode,
|
||||
workspaceRuntimeSupport,
|
||||
maintenanceBarrier,
|
||||
effectiveRelationships,
|
||||
});
|
||||
app.post("/internal/maintenance/activate", async (req, reply) => {
|
||||
try {
|
||||
await maintenanceBarrier.activate();
|
||||
return maintenanceBarrier.status();
|
||||
} catch {
|
||||
return reply.code(500).send({
|
||||
...maintenanceBarrier.status(),
|
||||
code: "maintenance_durability_failed",
|
||||
error: "maintenance activation durability was not acknowledged",
|
||||
});
|
||||
}
|
||||
});
|
||||
app.post("/internal/maintenance/deactivate", async (req, reply) => {
|
||||
try {
|
||||
maintenanceBarrier.deactivate();
|
||||
return maintenanceBarrier.status();
|
||||
} catch {
|
||||
return reply.code(500).send({
|
||||
...maintenanceBarrier.status(),
|
||||
code: "maintenance_durability_failed",
|
||||
error: "maintenance deactivation durability was not acknowledged",
|
||||
});
|
||||
}
|
||||
});
|
||||
app.get("/internal/maintenance/status", async (req, reply) => {
|
||||
return maintenanceBarrier.status();
|
||||
});
|
||||
sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
|
||||
metaRoutes(app, { harnessDir: config.harnessDir, listModels });
|
||||
workspaceRoutes(app, {
|
||||
registry: workspaceRegistry,
|
||||
config: config.workspaceRegistry,
|
||||
diagnose: workspaceDiagnoser,
|
||||
authDiagnoser,
|
||||
secretStore: workspaceSecretStore,
|
||||
});
|
||||
catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
|
||||
catalogTableRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
service: catalogTableService,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogSchemaRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: catalogSyncWorker,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogLogicalRelationshipRoutes(app, {
|
||||
service: catalogLogicalRelationshipService,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogDescriptionConsolidationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
metadataGenerationModelRoutes(app, metadataGenerationModels);
|
||||
catalogDescriptionGenerationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: descriptionGenerationWorker,
|
||||
sensitiveDataSuggestionRunner,
|
||||
});
|
||||
settingsRoutes(app, { cfg: config, listModels, getSettings });
|
||||
piManagementRoutes(app, { service: piManagement });
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
function corsOrigin(request: FastifyRequest, expectedOrigin: string): string | false {
|
||||
const supplied = request.headers.origin;
|
||||
if (typeof supplied !== "string") return false;
|
||||
try {
|
||||
return new URL(supplied).origin === expectedOrigin ? expectedOrigin : false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isLoopback(ip: string): boolean { return ip === "127.0.0.1" || ip === "::1" || ip === "::ffff:127.0.0.1"; }
|
||||
function isMaintenanceControl(url: string): boolean {
|
||||
return /^\/internal\/maintenance\/(?:activate|deactivate|status)(?:\?|$)/.test(url);
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
import type { FastifyRequest, FastifyReply, preHandlerHookHandler } from "fastify";
|
||||
import { localPrincipal, type PrincipalContext, upstreamPrincipal } from "./principal.js";
|
||||
import { rolesToPermissions } from "./config.js";
|
||||
import type { AuthenticationConfigProvider, AuthMode, AuthSessionRecord, LoadedAuthConfig } from "./types.js";
|
||||
import { AuthSessionOperationalError, type AuthSessionStore, type AuthSessionValidity } from "./session-store.js";
|
||||
import { deriveCsrfToken, csrfTokensEqual } from "./csrf.js";
|
||||
import { requireSameOriginOrNonBrowser } from "./authorization.js";
|
||||
|
||||
declare module "fastify" {
|
||||
interface FastifyRequest {
|
||||
principal?: PrincipalContext;
|
||||
authSession?: AuthSessionRecord;
|
||||
/** Internal only: never serialize or write this opaque cookie token to logs. */
|
||||
authSessionToken?: string;
|
||||
authPublicOrigin?: string;
|
||||
/** One immutable configuration load for the whole request, including CORS. */
|
||||
authConfigSnapshot?: LoadedAuthConfig;
|
||||
authConfigSnapshotCaptured?: boolean;
|
||||
authConfigSnapshotUnavailable?: boolean;
|
||||
}
|
||||
}
|
||||
|
||||
const SESSION_COOKIE = "thothii_session";
|
||||
const SESSION_TOKEN = /^[A-Za-z0-9_-]{43}$/;
|
||||
const STATE_CHANGING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
|
||||
|
||||
export interface AuthDependencies {
|
||||
mode: AuthMode;
|
||||
publicExposure?: boolean;
|
||||
authentication?: AuthenticationConfigProvider;
|
||||
sessionStore?: AuthSessionStore;
|
||||
sessionValidityForSnapshot?: (snapshot: LoadedAuthConfig) => AuthSessionValidity;
|
||||
}
|
||||
|
||||
/** Capture the authentication configuration once; CORS calls this before every other hook. */
|
||||
export function captureAuthConfigSnapshot(
|
||||
request: FastifyRequest,
|
||||
authentication: AuthenticationConfigProvider | undefined,
|
||||
): LoadedAuthConfig | undefined {
|
||||
if (request.authConfigSnapshotCaptured) return request.authConfigSnapshot;
|
||||
request.authConfigSnapshotCaptured = true;
|
||||
try {
|
||||
request.authConfigSnapshot = authentication?.current();
|
||||
} catch {
|
||||
request.authConfigSnapshotUnavailable = true;
|
||||
}
|
||||
return request.authConfigSnapshot;
|
||||
}
|
||||
|
||||
export function authPreHandler(mode: "none" | "mock" | "upstream", publicExposure = false) {
|
||||
return async (req: FastifyRequest, reply: FastifyReply) => {
|
||||
if (mode === "none") {
|
||||
req.principal = localPrincipal(publicExposure);
|
||||
} else if (mode === "mock") {
|
||||
const subject = typeof req.headers["x-mock-user"] === "string" ? req.headers["x-mock-user"].trim() : "mock";
|
||||
const elevated = req.headers["x-thoth-is-admin"] === "1" || req.headers["x-thoth-is-admin"] === "true";
|
||||
const roles = elevated ? ["admin"] as const : ["user"] as const;
|
||||
req.principal = {
|
||||
issuer: "mock", subject: subject || "mock", displayName: subject || "mock", roles,
|
||||
permissions: rolesToPermissions(roles), isAdmin: elevated,
|
||||
};
|
||||
} else {
|
||||
const principal = upstreamPrincipal(req.headers);
|
||||
if (!principal) {
|
||||
return reply.code(401).send({ error: "authenticated upstream identity required" });
|
||||
}
|
||||
req.principal = principal;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The one application boundary for principal resolution. Auth protocol endpoints are the only
|
||||
* public exceptions; all other routes get either a resolved principal or a sanitized denial.
|
||||
*/
|
||||
export function authenticateSession(deps: AuthDependencies): preHandlerHookHandler {
|
||||
const legacy = deps.mode === "none" || deps.mode === "mock" || deps.mode === "upstream"
|
||||
? authPreHandler(deps.mode, deps.publicExposure)
|
||||
: undefined;
|
||||
|
||||
const handle = async (request: FastifyRequest, reply: FastifyReply): Promise<void> => {
|
||||
const snapshot = captureAuthConfigSnapshot(request, deps.authentication);
|
||||
if (isPublicRoute(request)) return;
|
||||
|
||||
if (legacy) {
|
||||
await legacy(request, reply);
|
||||
if (reply.sent || !STATE_CHANGING_METHODS.has(request.method)) return;
|
||||
return requireSameOriginOrNonBrowser(request, reply);
|
||||
}
|
||||
|
||||
const origin = configuredOrigin(snapshot);
|
||||
if (!snapshot || !origin || !deps.sessionStore) {
|
||||
return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" });
|
||||
}
|
||||
const token = readSessionCookie(request);
|
||||
if (token === undefined || token === false) return authenticationRequired(reply);
|
||||
|
||||
let session: AuthSessionRecord | undefined;
|
||||
try {
|
||||
session = await deps.sessionStore.resolve(token, undefined, deps.sessionValidityForSnapshot?.(snapshot));
|
||||
if (session && session.authConfigRevision !== snapshot.revision) {
|
||||
try { await deps.sessionStore.revoke(token); } catch { /* the mismatch remains denied */ }
|
||||
return authenticationRequired(reply);
|
||||
}
|
||||
if (session) await deps.sessionStore.touch(token);
|
||||
} catch (error) {
|
||||
if (error instanceof AuthSessionOperationalError) {
|
||||
return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" });
|
||||
}
|
||||
return authenticationRequired(reply);
|
||||
}
|
||||
if (!session) return authenticationRequired(reply);
|
||||
|
||||
request.authSession = session;
|
||||
request.authSessionToken = token;
|
||||
request.authPublicOrigin = origin;
|
||||
request.principal = {
|
||||
issuer: session.issuer,
|
||||
subject: session.subject,
|
||||
...(session.displayName === undefined ? {} : { displayName: session.displayName }),
|
||||
roles: session.roles,
|
||||
permissions: session.permissions,
|
||||
isAdmin: session.roles.includes("admin"),
|
||||
};
|
||||
if (STATE_CHANGING_METHODS.has(request.method)) {
|
||||
requireCsrf(request, reply);
|
||||
return;
|
||||
}
|
||||
};
|
||||
return (request, reply, done) => {
|
||||
void handle(request, reply).then(
|
||||
() => done(),
|
||||
() => {
|
||||
if (!reply.sent) reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" });
|
||||
done();
|
||||
},
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
export function requireCsrf(request: FastifyRequest, reply: FastifyReply): true | FastifyReply {
|
||||
const expectedOrigin = request.authPublicOrigin;
|
||||
const token = request.authSessionToken;
|
||||
if (!expectedOrigin || !token) return authenticationRequired(reply);
|
||||
if (!matchesOrigin(request, expectedOrigin)) return csrfFailed(reply);
|
||||
|
||||
const header = singleHeader(request.headers["x-thothii-csrf"]);
|
||||
const supplied = header === false || header === undefined || !SESSION_TOKEN.test(header) ? undefined : header;
|
||||
let expected = "";
|
||||
try {
|
||||
expected = deriveCsrfToken(token);
|
||||
} catch {
|
||||
return authenticationRequired(reply);
|
||||
}
|
||||
if (!csrfTokensEqual(expected, supplied)) return csrfFailed(reply);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Require an exact configured public origin and browser Fetch Metadata when supplied. */
|
||||
export function requireExactOrigin(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
expectedOrigin: string,
|
||||
): true | FastifyReply {
|
||||
return matchesOrigin(request, expectedOrigin) ? true : csrfFailed(reply);
|
||||
}
|
||||
|
||||
export function sessionCookieName(): string { return SESSION_COOKIE; }
|
||||
|
||||
function authenticationRequired(reply: FastifyReply): FastifyReply {
|
||||
return reply.code(401).send({ code: "authentication_required", error: "Authentication is required" });
|
||||
}
|
||||
|
||||
function csrfFailed(reply: FastifyReply): FastifyReply {
|
||||
return reply.code(403).send({ code: "csrf_failed", error: "Request origin validation failed" });
|
||||
}
|
||||
|
||||
export function configuredOrigin(snapshot: LoadedAuthConfig | undefined): string | undefined {
|
||||
try {
|
||||
const publicUrl = snapshot?.value.publicUrl;
|
||||
return publicUrl ? new URL(publicUrl).origin : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function readSessionCookie(request: FastifyRequest): string | false | undefined {
|
||||
const raw = request.headers.cookie;
|
||||
if (raw === undefined) return undefined;
|
||||
if (Array.isArray(raw) || typeof raw !== "string" || raw.length > 4096) return false;
|
||||
const values = raw.split(";").filter((part) => /^\s*thothii_session(?:=|\s*$)/.test(part));
|
||||
if (values.length !== 1) return values.length === 0 ? undefined : false;
|
||||
const match = /^\s*thothii_session=([A-Za-z0-9_-]{43})\s*$/.exec(values[0]);
|
||||
return match?.[1] ?? false;
|
||||
}
|
||||
|
||||
function singleHeader(value: string | string[] | undefined): string | false | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (Array.isArray(value) || typeof value !== "string" || value.includes(",")) return false;
|
||||
return value;
|
||||
}
|
||||
|
||||
function matchesOrigin(request: FastifyRequest, expectedOrigin: string): boolean {
|
||||
const origin = singleHeader(request.headers.origin);
|
||||
try {
|
||||
if (origin === undefined || origin === false || new URL(origin).origin !== expectedOrigin) return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
const fetchSite = singleHeader(request.headers["sec-fetch-site"]);
|
||||
return fetchSite === undefined || fetchSite === "same-origin";
|
||||
}
|
||||
|
||||
function isPublicRoute(request: FastifyRequest): boolean {
|
||||
const rawUrl = request.raw.url ?? request.url;
|
||||
const query = rawUrl.indexOf("?");
|
||||
const pathname = query === -1 ? rawUrl : rawUrl.slice(0, query);
|
||||
return (request.method === "GET" && (pathname === "/health" || pathname === "/auth/config"
|
||||
|| pathname === "/auth/oidc/login" || pathname === "/auth/oidc/callback"))
|
||||
|| (request.method === "POST" && pathname === "/auth/local/login");
|
||||
}
|
||||
|
||||
export function getPrincipal(req: FastifyRequest): PrincipalContext {
|
||||
if (!req.principal) throw new Error("principal missing after authentication");
|
||||
return req.principal;
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
import type { AuthDiagnostic, GroupCatalog } from "./group-catalog.js";
|
||||
import { isUsableAuthenticationSecret } from "./secret-policy.js";
|
||||
import { parseConfiguredTransportUrl } from "./url-policy.js";
|
||||
|
||||
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
||||
const REQUEST_TIMEOUT_MS = 5_000;
|
||||
|
||||
export interface AuthentikGroupCatalogOptions {
|
||||
baseUrl: string;
|
||||
apiToken: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
function diagnostic(
|
||||
code: AuthDiagnostic["code"],
|
||||
message: string,
|
||||
field?: string,
|
||||
): AuthDiagnostic {
|
||||
return { level: "error", code, message, ...(field === undefined ? {} : { field }) };
|
||||
}
|
||||
|
||||
function catalogUnreachable(): AuthDiagnostic {
|
||||
return diagnostic("oidc_group_catalog_unreachable", "The configured group catalog is unavailable.");
|
||||
}
|
||||
|
||||
function catalogUnauthorized(): AuthDiagnostic {
|
||||
return diagnostic("oidc_group_catalog_unauthorized", "The configured group catalog credentials were rejected.");
|
||||
}
|
||||
|
||||
function missing(name: string): AuthDiagnostic {
|
||||
return diagnostic("oidc_mapped_group_missing", "A configured authorization group does not exist.", name);
|
||||
}
|
||||
|
||||
function ambiguous(name: string): AuthDiagnostic {
|
||||
return diagnostic("oidc_mapped_group_ambiguous", "A configured authorization group is ambiguous.", name);
|
||||
}
|
||||
|
||||
function stableCompare(left: string, right: string): number {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function abortReason(signal: AbortSignal): unknown {
|
||||
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
||||
}
|
||||
|
||||
function cancelResponse(response: Response): void {
|
||||
try {
|
||||
const cancelled = response.body?.cancel();
|
||||
if (cancelled) void cancelled.catch(() => undefined);
|
||||
} catch { /* cancellation is advisory and never changes the diagnostic */ }
|
||||
}
|
||||
|
||||
function cancelReader(reader: ReadableStreamDefaultReader<Uint8Array>): void {
|
||||
try {
|
||||
const cancelled = reader.cancel();
|
||||
void cancelled.catch(() => undefined);
|
||||
} catch { /* cancellation is advisory and never changes the diagnostic */ }
|
||||
}
|
||||
|
||||
function awaitWithAbort<T>(
|
||||
operation: Promise<T>,
|
||||
signal: AbortSignal,
|
||||
onLateResolution?: (value: T) => void,
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const abort = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(abortReason(signal));
|
||||
};
|
||||
if (signal.aborted) {
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
operation.then(
|
||||
(value) => {
|
||||
if (settled) {
|
||||
try { onLateResolution?.(value); } catch { /* best-effort cleanup only */ }
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function validContentLength(response: Response): boolean {
|
||||
const value = response.headers.get("content-length");
|
||||
if (value === null) return true;
|
||||
if (!/^\d+$/.test(value)) return false;
|
||||
const length = Number(value);
|
||||
return Number.isSafeInteger(length) && length <= MAX_RESPONSE_BYTES;
|
||||
}
|
||||
|
||||
async function readBounded(response: Response, signal: AbortSignal): Promise<Uint8Array | undefined> {
|
||||
if (!validContentLength(response)) {
|
||||
cancelResponse(response);
|
||||
return undefined;
|
||||
}
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return new Uint8Array();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
let complete = false;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await awaitWithAbort(reader.read(), signal);
|
||||
if (done) break;
|
||||
if (value.byteLength > MAX_RESPONSE_BYTES - size) return undefined;
|
||||
chunks.push(value);
|
||||
size += value.byteLength;
|
||||
}
|
||||
complete = true;
|
||||
const body = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
body.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return body;
|
||||
} finally {
|
||||
if (!complete) cancelReader(reader);
|
||||
try { reader.releaseLock(); } catch { /* reader may already be unusable */ }
|
||||
}
|
||||
}
|
||||
|
||||
type GroupResult = "present" | "missing" | "ambiguous" | "unauthorized" | "unreachable";
|
||||
|
||||
function exactResult(name: string, parsed: unknown): GroupResult {
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return "unreachable";
|
||||
const record = parsed as { results?: unknown; pagination?: unknown };
|
||||
if (!Array.isArray(record.results) || record.results.length > 2
|
||||
|| !record.pagination || typeof record.pagination !== "object"
|
||||
|| Array.isArray(record.pagination)) return "unreachable";
|
||||
if (!Object.prototype.hasOwnProperty.call(record.pagination, "next")) return "unreachable";
|
||||
const next = (record.pagination as { next: unknown }).next;
|
||||
if (next !== null) {
|
||||
if (typeof next !== "string" || next.length === 0 || next.length > 2048 || /\p{Cc}/u.test(next)) return "unreachable";
|
||||
try {
|
||||
const continuation = new URL(next);
|
||||
if (continuation.protocol !== "https:" || continuation.username || continuation.password || continuation.hash) return "unreachable";
|
||||
} catch {
|
||||
return "unreachable";
|
||||
}
|
||||
return "ambiguous";
|
||||
}
|
||||
const resultNames: string[] = [];
|
||||
for (const result of record.results) {
|
||||
if (!result || typeof result !== "object" || Array.isArray(result)
|
||||
|| typeof (result as { name?: unknown }).name !== "string") return "unreachable";
|
||||
resultNames.push((result as { name: string }).name);
|
||||
}
|
||||
const exactMatches = resultNames.filter((candidate) => candidate === name).length;
|
||||
if (exactMatches === 0) return "missing";
|
||||
return exactMatches === 1 ? "present" : "ambiguous";
|
||||
}
|
||||
|
||||
export function createAuthentikGroupCatalog(options: AuthentikGroupCatalogOptions): GroupCatalog {
|
||||
const origin = parseConfiguredTransportUrl(options.baseUrl, { allowLoopbackHttp: false, originOnly: true });
|
||||
const fetchImplementation = options.fetch ?? globalThis.fetch;
|
||||
const valid = origin !== undefined
|
||||
&& isUsableAuthenticationSecret("THT_AUTHENTIK_API_TOKEN", options.apiToken)
|
||||
&& typeof fetchImplementation === "function";
|
||||
|
||||
async function verify(name: string, signal: AbortSignal): Promise<GroupResult> {
|
||||
if (!origin || !valid || signal.aborted) return "unreachable";
|
||||
const target = new URL("/api/v3/core/groups/", origin);
|
||||
target.searchParams.set("name", name);
|
||||
target.searchParams.set("include_users", "false");
|
||||
target.searchParams.set("page_size", "2");
|
||||
const timeout = new AbortController();
|
||||
const timer = setTimeout(() => timeout.abort(), REQUEST_TIMEOUT_MS);
|
||||
timer.unref();
|
||||
const requestSignal = AbortSignal.any([signal, timeout.signal]);
|
||||
try {
|
||||
const response = await awaitWithAbort(
|
||||
Promise.resolve().then(() => fetchImplementation(target, {
|
||||
headers: { accept: "application/json", authorization: `Bearer ${options.apiToken}` },
|
||||
redirect: "error",
|
||||
signal: requestSignal,
|
||||
})),
|
||||
requestSignal,
|
||||
cancelResponse,
|
||||
);
|
||||
if (response.redirected || response.type === "opaqueredirect" || response.status >= 300 && response.status < 400) {
|
||||
cancelResponse(response);
|
||||
return "unreachable";
|
||||
}
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
cancelResponse(response);
|
||||
return "unauthorized";
|
||||
}
|
||||
if (!response.ok) {
|
||||
cancelResponse(response);
|
||||
return "unreachable";
|
||||
}
|
||||
const body = await readBounded(response, requestSignal);
|
||||
if (body === undefined) return "unreachable";
|
||||
try {
|
||||
return exactResult(name, JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(body)));
|
||||
} catch {
|
||||
return "unreachable";
|
||||
}
|
||||
} catch {
|
||||
return "unreachable";
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async verifyConfiguredGroups(names, signal) {
|
||||
const diagnostics: AuthDiagnostic[] = [];
|
||||
for (const name of [...new Set(names)].sort(stableCompare)) {
|
||||
const outcome = await verify(name, signal);
|
||||
if (outcome === "present") continue;
|
||||
if (outcome === "missing") diagnostics.push(missing(name));
|
||||
else if (outcome === "ambiguous") diagnostics.push(ambiguous(name));
|
||||
else if (outcome === "unauthorized") return [catalogUnauthorized()];
|
||||
else return [catalogUnreachable()];
|
||||
}
|
||||
return diagnostics;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
import type { FastifyReply, FastifyRequest } from "fastify";
|
||||
import type { Permission } from "./types.js";
|
||||
import { getPrincipal } from "./auth.js";
|
||||
import type { PrincipalContext } from "./principal.js";
|
||||
|
||||
export function hasPermission(principal: PrincipalContext, permission: Permission): boolean {
|
||||
return principal.permissions.includes(permission);
|
||||
}
|
||||
|
||||
export function isPrincipalContext(
|
||||
value: PrincipalContext | FastifyReply,
|
||||
): value is PrincipalContext {
|
||||
return "issuer" in value;
|
||||
}
|
||||
|
||||
export function requirePermission(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
permission: Permission,
|
||||
): PrincipalContext | FastifyReply {
|
||||
const principal = getPrincipal(request);
|
||||
if (hasPermission(principal, permission)) return principal;
|
||||
return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" });
|
||||
}
|
||||
|
||||
/**
|
||||
* A resolved cookie session is populated only by the central auth boundary, after its
|
||||
* request-snapshot Origin and CSRF checks. Route-specific legacy guards must not reinterpret
|
||||
* the internal transport host/protocol for that already-authorized browser request.
|
||||
*/
|
||||
export function hasCookieBackedAuthSession(request: FastifyRequest): boolean {
|
||||
return request.authSession !== undefined;
|
||||
}
|
||||
|
||||
/** Permit non-browser clients and browsers whose declared origin matches the request host. */
|
||||
export function requireSameOriginOrNonBrowser(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
): FastifyReply | undefined {
|
||||
if (hasCookieBackedAuthSession(request)) return undefined;
|
||||
const origin = request.headers.origin;
|
||||
if (origin === undefined) return undefined;
|
||||
if (typeof origin !== "string" || typeof request.headers.host !== "string") {
|
||||
return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" });
|
||||
}
|
||||
try {
|
||||
const supplied = new URL(origin);
|
||||
const expected = new URL(`${request.protocol}://${request.headers.host}`);
|
||||
if (supplied.origin === expected.origin) return undefined;
|
||||
} catch {
|
||||
// Invalid browser origins are forbidden below.
|
||||
}
|
||||
return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" });
|
||||
}
|
||||
@@ -1,320 +0,0 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
closeSync,
|
||||
constants,
|
||||
fstatSync,
|
||||
lstatSync,
|
||||
openSync,
|
||||
readSync,
|
||||
realpathSync,
|
||||
} from "node:fs";
|
||||
import type { Stats } from "node:fs";
|
||||
import { dirname, isAbsolute, normalize } from "node:path";
|
||||
import { parseDocument } from "yaml";
|
||||
import { z } from "zod";
|
||||
import type {
|
||||
AuthenticationConfig,
|
||||
AuthenticationConfigProvider,
|
||||
AuthMode,
|
||||
LoadedAuthConfig,
|
||||
Permission,
|
||||
Role,
|
||||
} from "./types.js";
|
||||
import { parseConfiguredTransportUrl } from "./url-policy.js";
|
||||
import { createWindowsAuthStorageBridge, type WindowsAuthStorageBridge } from "./windows-auth-storage.js";
|
||||
|
||||
export type {
|
||||
AuthenticationConfig,
|
||||
AuthenticationConfigProvider,
|
||||
AuthMode,
|
||||
LoadedAuthConfig,
|
||||
Permission,
|
||||
Role,
|
||||
} from "./types.js";
|
||||
|
||||
const MAX_AUTH_CONFIG_BYTES = 1024 * 1024;
|
||||
// Keep live catalog work within the same deterministic bound as the mandatory direct groups claim.
|
||||
const MAX_MAPPED_GROUPS = 128;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
export const PERMISSION_CATALOG: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
const invalid = (): Error => new Error("authentication configuration is invalid");
|
||||
const nonEmptyText = z.string().min(1).max(512).refine(
|
||||
(value) => value.trim() === value && !/[\u0000-\u001f\u007f]/.test(value),
|
||||
);
|
||||
const positiveSeconds = z.number().int().min(1).max(365 * 24 * 60 * 60);
|
||||
const sessionSchema = z.strictObject({
|
||||
regularTtlSeconds: positiveSeconds.default(43_200),
|
||||
regularIdleSeconds: positiveSeconds.default(7_200),
|
||||
rememberTtlSeconds: positiveSeconds.default(2_592_000),
|
||||
rememberIdleSeconds: positiveSeconds.default(604_800),
|
||||
oidcTtlSeconds: positiveSeconds.default(28_800),
|
||||
});
|
||||
const roleSchema = z.enum(ROLES);
|
||||
const groupNameSchema = nonEmptyText.max(256);
|
||||
const groupRolesSchema = z.record(groupNameSchema, z.array(roleSchema).min(1))
|
||||
.refine((value) => Object.keys(value).length <= MAX_MAPPED_GROUPS);
|
||||
|
||||
const localSchema = z.strictObject({
|
||||
version: z.literal(1), mode: z.literal("local"), publicUrl: nonEmptyText, session: sessionSchema.optional(),
|
||||
local: z.strictObject({ usersFile: nonEmptyText.max(255) }),
|
||||
});
|
||||
const oidcSchema = z.strictObject({
|
||||
version: z.literal(1), mode: z.literal("oidc"), publicUrl: nonEmptyText, session: sessionSchema.optional(),
|
||||
oidc: z.strictObject({
|
||||
issuer: nonEmptyText, clientId: nonEmptyText, clientSecretRef: z.literal("THT_OIDC_CLIENT_SECRET"),
|
||||
scopes: z.array(nonEmptyText).min(1).max(16), groupsClaim: z.literal("groups"),
|
||||
}),
|
||||
groupCatalog: z.strictObject({
|
||||
driver: z.literal("authentik"), baseUrl: nonEmptyText, apiTokenRef: z.literal("THT_AUTHENTIK_API_TOKEN"),
|
||||
}),
|
||||
authorization: z.strictObject({ groupRoles: groupRolesSchema }),
|
||||
});
|
||||
|
||||
interface FileIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
ctimeMs: number;
|
||||
mode: number;
|
||||
nlink: number;
|
||||
}
|
||||
|
||||
interface DirectoryIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
mode: number;
|
||||
ctimeMs: number;
|
||||
}
|
||||
|
||||
interface StorageIdentity {
|
||||
file: FileIdentity;
|
||||
directory: DirectoryIdentity;
|
||||
}
|
||||
|
||||
export interface AuthenticationConfigLoadOptions {
|
||||
/** Test seam; production creates the existing bounded internal tht auth-storage bridge. */
|
||||
windowsStorageBridge?: Pick<WindowsAuthStorageBridge, "readAuthConfig">;
|
||||
}
|
||||
|
||||
function validateCanonicalPath(path: string): void {
|
||||
if (typeof path !== "string" || path.length === 0 || path.trim() !== path
|
||||
|| path.includes("\0") || !isAbsolute(path) || normalize(path) !== path
|
||||
|| realpathSync(path) !== path || realpathSync(dirname(path)) !== dirname(path)) throw invalid();
|
||||
}
|
||||
|
||||
function runtimeOwner(): number {
|
||||
if (process.platform === "win32" || typeof process.geteuid !== "function") throw invalid();
|
||||
const owner = process.geteuid();
|
||||
if (!Number.isSafeInteger(owner) || owner < 0) throw invalid();
|
||||
return owner;
|
||||
}
|
||||
|
||||
function fileMetadata(info: Stats): FileIdentity {
|
||||
const mode = info.mode & 0o7777;
|
||||
if (!info.isFile() || info.uid !== runtimeOwner() || info.nlink !== 1 || mode !== 0o600
|
||||
|| info.size < 0 || info.size > MAX_AUTH_CONFIG_BYTES) throw invalid();
|
||||
return {
|
||||
dev: info.dev, ino: info.ino, uid: info.uid, size: info.size,
|
||||
mtimeMs: info.mtimeMs, ctimeMs: info.ctimeMs, mode, nlink: info.nlink,
|
||||
};
|
||||
}
|
||||
|
||||
function directoryMetadata(info: Stats): DirectoryIdentity {
|
||||
const mode = info.mode & 0o7777;
|
||||
if (!info.isDirectory() || info.uid !== runtimeOwner() || mode !== 0o700) throw invalid();
|
||||
return { dev: info.dev, ino: info.ino, uid: info.uid, mode, ctimeMs: info.ctimeMs };
|
||||
}
|
||||
|
||||
function sameFileIdentity(left: FileIdentity, right: FileIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid
|
||||
&& left.size === right.size && left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs
|
||||
&& left.mode === right.mode && left.nlink === right.nlink;
|
||||
}
|
||||
|
||||
function sameDirectoryIdentity(left: DirectoryIdentity, right: DirectoryIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid
|
||||
&& left.mode === right.mode && left.ctimeMs === right.ctimeMs;
|
||||
}
|
||||
|
||||
function sameIdentity(left: StorageIdentity, right: StorageIdentity): boolean {
|
||||
return sameFileIdentity(left.file, right.file) && sameDirectoryIdentity(left.directory, right.directory);
|
||||
}
|
||||
|
||||
function storageIdentity(path: string): StorageIdentity {
|
||||
try {
|
||||
validateCanonicalPath(path);
|
||||
return {
|
||||
file: fileMetadata(lstatSync(path) as Stats),
|
||||
directory: directoryMetadata(lstatSync(dirname(path)) as Stats),
|
||||
};
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function openDirectoryDescriptor(path: string): number {
|
||||
return openSync(path, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0)
|
||||
| (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0));
|
||||
}
|
||||
|
||||
function readBoundedConfig(path: string): { source: string; identity: StorageIdentity } {
|
||||
let directoryDescriptor: number | undefined;
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
const before = storageIdentity(path);
|
||||
directoryDescriptor = openDirectoryDescriptor(dirname(path));
|
||||
const openedDirectory = directoryMetadata(fstatSync(directoryDescriptor) as Stats);
|
||||
if (!sameDirectoryIdentity(before.directory, openedDirectory)) throw invalid();
|
||||
fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
const opened = fileMetadata(fstatSync(fd) as Stats);
|
||||
if (!sameFileIdentity(before.file, opened)) throw invalid();
|
||||
const buffer = Buffer.allocUnsafe(MAX_AUTH_CONFIG_BYTES + 1);
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const bytesRead = readSync(fd, buffer, offset, buffer.length - offset, null);
|
||||
if (bytesRead === 0) break;
|
||||
offset += bytesRead;
|
||||
}
|
||||
if (offset > MAX_AUTH_CONFIG_BYTES) throw invalid();
|
||||
const afterFile = fileMetadata(fstatSync(fd) as Stats);
|
||||
const afterPath = storageIdentity(path);
|
||||
const afterOpenedDirectory = directoryMetadata(fstatSync(directoryDescriptor) as Stats);
|
||||
if (!sameFileIdentity(opened, afterFile) || !sameFileIdentity(afterFile, afterPath.file)
|
||||
|| !sameDirectoryIdentity(before.directory, afterPath.directory)
|
||||
|| !sameDirectoryIdentity(openedDirectory, afterOpenedDirectory)) throw invalid();
|
||||
validateCanonicalPath(path);
|
||||
return {
|
||||
source: new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, offset)),
|
||||
identity: afterPath,
|
||||
};
|
||||
} catch {
|
||||
throw invalid();
|
||||
} finally {
|
||||
if (fd !== undefined) try { closeSync(fd); } catch { /* sanitized by design */ }
|
||||
if (directoryDescriptor !== undefined) try { closeSync(directoryDescriptor); } catch { /* sanitized by design */ }
|
||||
}
|
||||
}
|
||||
|
||||
function validOrigin(value: string, httpLoopbackAllowed: boolean): boolean {
|
||||
return parseConfiguredTransportUrl(value, { allowLoopbackHttp: httpLoopbackAllowed, originOnly: true }) !== undefined;
|
||||
}
|
||||
|
||||
function validIssuer(value: string): boolean {
|
||||
return parseConfiguredTransportUrl(value, { allowLoopbackHttp: false }) !== undefined;
|
||||
}
|
||||
|
||||
function validUsersFile(value: string): boolean {
|
||||
return /^[A-Za-z0-9][A-Za-z0-9._-]*\.yaml$/.test(value);
|
||||
}
|
||||
|
||||
function canonicalize(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.entries(value as Record<string, unknown>)
|
||||
.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)
|
||||
.map(([key, nested]) => [key, canonicalize(nested)]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function canonicalRevision(value: AuthenticationConfig): string {
|
||||
return createHash("sha256").update(JSON.stringify(canonicalize(value))).digest("hex");
|
||||
}
|
||||
|
||||
export function parseAuthenticationConfigSource(source: string): AuthenticationConfig {
|
||||
try {
|
||||
const document = parseDocument(source, { uniqueKeys: true });
|
||||
if (document.errors.length > 0 || document.warnings.length > 0) throw invalid();
|
||||
const parsed = document.toJSON();
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw invalid();
|
||||
const config = parsed as Record<string, unknown>;
|
||||
const schema = config.mode === "local" ? localSchema : config.mode === "oidc" ? oidcSchema : undefined;
|
||||
if (!schema) throw invalid();
|
||||
const validated = schema.parse(config);
|
||||
const session = sessionSchema.parse(validated.session ?? {});
|
||||
if (!validOrigin(validated.publicUrl, true)) throw invalid();
|
||||
if (validated.mode === "local") {
|
||||
if (!validUsersFile(validated.local.usersFile)) throw invalid();
|
||||
return { ...validated, session };
|
||||
}
|
||||
if (!validIssuer(validated.oidc.issuer) || !validOrigin(validated.groupCatalog.baseUrl, false)) throw invalid();
|
||||
if (!validated.oidc.scopes.includes("openid")) throw invalid();
|
||||
const mappings = Object.entries(validated.authorization.groupRoles);
|
||||
if (mappings.length === 0 || mappings.filter(([, roles]) => roles.includes("admin")).length !== 1) throw invalid();
|
||||
return { ...validated, session };
|
||||
} catch { throw invalid(); }
|
||||
}
|
||||
|
||||
function loadAuthenticationConfigWithIdentity(path: string): { loaded: LoadedAuthConfig; identity: StorageIdentity } {
|
||||
const read = readBoundedConfig(path);
|
||||
const value = parseAuthenticationConfigSource(read.source);
|
||||
return { loaded: { value, revision: canonicalRevision(value), sourcePath: path }, identity: read.identity };
|
||||
}
|
||||
|
||||
function loadWindowsAuthenticationConfig(
|
||||
path: string,
|
||||
bridge: Pick<WindowsAuthStorageBridge, "readAuthConfig">,
|
||||
): LoadedAuthConfig {
|
||||
try {
|
||||
const contents = bridge.readAuthConfig(path);
|
||||
if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > MAX_AUTH_CONFIG_BYTES) throw invalid();
|
||||
const source = new TextDecoder("utf-8", { fatal: true }).decode(contents);
|
||||
const value = parseAuthenticationConfigSource(source);
|
||||
return { value, revision: canonicalRevision(value), sourcePath: path };
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
export function loadAuthenticationConfig(path: string, options: AuthenticationConfigLoadOptions = {}): LoadedAuthConfig {
|
||||
if (process.platform === "win32") {
|
||||
return loadWindowsAuthenticationConfig(path, options.windowsStorageBridge ?? createWindowsAuthStorageBridge());
|
||||
}
|
||||
return loadAuthenticationConfigWithIdentity(path).loaded;
|
||||
}
|
||||
|
||||
export function createAuthenticationConfigProvider(
|
||||
path: string,
|
||||
options: AuthenticationConfigLoadOptions = {},
|
||||
): AuthenticationConfigProvider {
|
||||
if (process.platform === "win32") {
|
||||
const bridge = options.windowsStorageBridge ?? createWindowsAuthStorageBridge();
|
||||
return { current: () => loadWindowsAuthenticationConfig(path, bridge) };
|
||||
}
|
||||
let cached: { identity: StorageIdentity; loaded: LoadedAuthConfig } | undefined;
|
||||
return { current(): LoadedAuthConfig {
|
||||
const before = storageIdentity(path);
|
||||
if (cached && sameIdentity(cached.identity, before)) return cached.loaded;
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
const { loaded, identity } = loadAuthenticationConfigWithIdentity(path);
|
||||
if (sameIdentity(identity, storageIdentity(path))) {
|
||||
cached = { identity, loaded };
|
||||
return loaded;
|
||||
}
|
||||
} catch { /* retry one concurrent atomic replacement, then fail closed */ }
|
||||
}
|
||||
throw invalid();
|
||||
} };
|
||||
}
|
||||
|
||||
export function rolesToPermissions(roles: readonly Role[]): readonly Permission[] {
|
||||
const requested = new Set<Role>();
|
||||
for (const role of roles) {
|
||||
if (!ROLES.includes(role)) throw invalid();
|
||||
requested.add(role);
|
||||
}
|
||||
if (requested.has("admin")) return PERMISSION_CATALOG;
|
||||
return requested.has("user") ? ["session.use"] : [];
|
||||
}
|
||||
|
||||
export function isPermission(value: string): value is Permission {
|
||||
return PERMISSION_CATALOG.includes(value as Permission);
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import { timingSafeEqual } from "node:crypto";
|
||||
import { deriveCsrfToken as deriveStoredCsrfToken } from "./session-store.js";
|
||||
|
||||
export { deriveStoredCsrfToken as deriveCsrfToken };
|
||||
|
||||
/** Compare a client-supplied CSRF value without exposing a useful length timing oracle. */
|
||||
export function csrfTokensEqual(expectedToken: string, suppliedToken: string | undefined): boolean {
|
||||
const expected = Buffer.from(expectedToken, "utf8");
|
||||
const supplied = Buffer.from(suppliedToken ?? "", "utf8");
|
||||
const padded = Buffer.alloc(expected.length);
|
||||
supplied.copy(padded, 0, 0, expected.length);
|
||||
return timingSafeEqual(expected, padded) && supplied.length === expected.length;
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolve } from "node:path";
|
||||
import {
|
||||
closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
|
||||
} from "node:fs";
|
||||
import { loadConfig, type AppConfig } from "../config.js";
|
||||
import { loadSecretBundle, secretValue } from "../config/secret-bundle.js";
|
||||
import { createAuthentikGroupCatalog } from "./authentik-group-catalog.js";
|
||||
import { createCurrentLocalUserRegistryResolver, type LocalUserRegistry } from "./local-registry.js";
|
||||
import { createOidcProtocol, OidcDeviceFlowUnavailableError, type OidcProtocol } from "./oidc-client.js";
|
||||
import { createAuthDiagnoser, type AuthDiagnoser, type AuthDiagnostic, type AuthDiagnostics } from "./diagnostics.js";
|
||||
import { decodeAuthDiagnostics, type GroupCatalog } from "./group-catalog.js";
|
||||
import type { LoadedAuthConfig } from "./types.js";
|
||||
|
||||
const AUTH_SECRET_REFERENCES = ["THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN"] as const;
|
||||
const MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES = 64 * 1024;
|
||||
const MAX_DIAGNOSTIC_SECRET_VALUES = 4096;
|
||||
const MAX_DIAGNOSTIC_SECRET_DEPTH = 32;
|
||||
|
||||
function unavailableSecretCorpus(): Error {
|
||||
return new Error("diagnostic secret corpus is unavailable");
|
||||
}
|
||||
|
||||
function readMountedSecretSource(file: string): string {
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
if (!file || file.trim() !== file || file.includes("\0")) throw unavailableSecretCorpus();
|
||||
const before = lstatSync(file);
|
||||
if (!before.isFile() || before.isSymbolicLink() || before.size > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const opened = fstatSync(fd);
|
||||
if (!opened.isFile() || opened.size > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES
|
||||
|| before.dev !== opened.dev || before.ino !== opened.ino) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
const value = readFileSync(fd, "utf8");
|
||||
if (Buffer.byteLength(value, "utf8") > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
return value;
|
||||
} catch {
|
||||
throw unavailableSecretCorpus();
|
||||
} finally {
|
||||
if (fd !== undefined) try { closeSync(fd); } catch { /* fixed failure surface above */ }
|
||||
}
|
||||
}
|
||||
|
||||
function parsedSecretValues(raw: string, requireJson: boolean): readonly string[] {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return [];
|
||||
const values = new Set<string>([raw.replace(/[\r\n]+$/u, "")]);
|
||||
const looksJson = trimmed.startsWith("{") || trimmed.startsWith("[");
|
||||
if (!looksJson) {
|
||||
if (requireJson) throw unavailableSecretCorpus();
|
||||
return [...values];
|
||||
}
|
||||
let document: unknown;
|
||||
try { document = JSON.parse(trimmed); } catch { throw unavailableSecretCorpus(); }
|
||||
if (requireJson && (!document || typeof document !== "object" || Array.isArray(document))) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
const pending: Array<{ value: unknown; depth: number }> = [{ value: document, depth: 0 }];
|
||||
let scalarCount = 0;
|
||||
while (pending.length > 0) {
|
||||
const current = pending.pop()!;
|
||||
if (current.depth > MAX_DIAGNOSTIC_SECRET_DEPTH) throw unavailableSecretCorpus();
|
||||
if (Array.isArray(current.value)) {
|
||||
for (const item of current.value) pending.push({ value: item, depth: current.depth + 1 });
|
||||
} else if (current.value && typeof current.value === "object") {
|
||||
for (const item of Object.values(current.value as Record<string, unknown>)) {
|
||||
pending.push({ value: item, depth: current.depth + 1 });
|
||||
}
|
||||
} else {
|
||||
scalarCount += 1;
|
||||
if (scalarCount > 1024) throw unavailableSecretCorpus();
|
||||
if (typeof current.value === "string" && current.value.length > 0) values.add(current.value);
|
||||
}
|
||||
}
|
||||
return [...values];
|
||||
}
|
||||
|
||||
export function configuredSecretValues(config: AppConfig): readonly string[] {
|
||||
try {
|
||||
const values = new Set<string>();
|
||||
if (config.secretsFile) {
|
||||
for (const value of loadSecretBundle(config.secretsFile).values()) values.add(value);
|
||||
}
|
||||
const legacyFiles = new Set(Object.values(config.secretFiles).filter(
|
||||
(file): file is string => file !== undefined,
|
||||
));
|
||||
for (const file of legacyFiles) {
|
||||
for (const value of parsedSecretValues(readMountedSecretSource(file), false)) values.add(value);
|
||||
}
|
||||
if (config.piAuthFile) {
|
||||
for (const value of parsedSecretValues(readMountedSecretSource(config.piAuthFile), true)) values.add(value);
|
||||
}
|
||||
if (values.size > MAX_DIAGNOSTIC_SECRET_VALUES) throw unavailableSecretCorpus();
|
||||
return [...values];
|
||||
} catch {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
}
|
||||
|
||||
export interface ConfiguredAuthDiagnoserOptions {
|
||||
localUserRegistry?: (loaded: LoadedAuthConfig) => LocalUserRegistry | undefined;
|
||||
oidcProtocol?: (loaded: LoadedAuthConfig) => OidcProtocol | undefined;
|
||||
groupCatalog?: (loaded: LoadedAuthConfig) => GroupCatalog | undefined;
|
||||
sessionRootValidator?: (root: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
/** Builds the one shared auth diagnostic implementation used by app routes and the one-shot CLI. */
|
||||
export function createConfiguredAuthDiagnoser(
|
||||
config: AppConfig,
|
||||
options: ConfiguredAuthDiagnoserOptions = {},
|
||||
): AuthDiagnoser {
|
||||
const localResolver = options.localUserRegistry === undefined
|
||||
? createCurrentLocalUserRegistryResolver()
|
||||
: undefined;
|
||||
const secretValues = (): ReadonlyMap<string, string> => {
|
||||
const values = new Map<string, string>();
|
||||
for (const reference of AUTH_SECRET_REFERENCES) {
|
||||
try {
|
||||
const value = secretValue(config, reference);
|
||||
if (value !== undefined) values.set(reference, value);
|
||||
} catch {
|
||||
// The shared diagnoser emits the fixed missing-secret diagnostic below.
|
||||
}
|
||||
}
|
||||
return values;
|
||||
};
|
||||
const loaded = (): LoadedAuthConfig | undefined => {
|
||||
try { return config.authentication?.current(); } catch { return undefined; }
|
||||
};
|
||||
return {
|
||||
async inspect(request): Promise<AuthDiagnostics> {
|
||||
const current = loaded();
|
||||
const protocol = current?.value.mode === "oidc"
|
||||
? options.oidcProtocol?.(current) ?? (() => {
|
||||
try {
|
||||
const clientSecret = secretValues().get("THT_OIDC_CLIENT_SECRET");
|
||||
if (!clientSecret) return undefined;
|
||||
return createOidcProtocol({
|
||||
issuer: current.value.oidc.issuer,
|
||||
clientId: current.value.oidc.clientId,
|
||||
clientSecret,
|
||||
callbackUrl: new URL("/api/auth/oidc/callback", current.value.publicUrl).href,
|
||||
scopes: current.value.oidc.scopes,
|
||||
groupsClaim: current.value.oidc.groupsClaim,
|
||||
});
|
||||
} catch { return undefined; }
|
||||
})()
|
||||
: undefined;
|
||||
const groupCatalog = current?.value.mode === "oidc" ? options.groupCatalog?.(current) ?? (() => {
|
||||
try {
|
||||
const token = secretValues().get("THT_AUTHENTIK_API_TOKEN");
|
||||
return token === undefined ? undefined : createAuthentikGroupCatalog({
|
||||
baseUrl: current.value.groupCatalog.baseUrl,
|
||||
apiToken: token,
|
||||
});
|
||||
} catch { return undefined; }
|
||||
})() : undefined;
|
||||
const report = await createAuthDiagnoser({
|
||||
authMode: config.authMode,
|
||||
authStateRoot: config.authStateRoot,
|
||||
...(options.sessionRootValidator === undefined ? {} : { sessionRootValidator: options.sessionRootValidator }),
|
||||
authentication: config.authentication,
|
||||
secrets: secretValues(),
|
||||
localUserRegistry: current?.value.mode === "local"
|
||||
? options.localUserRegistry?.(current) ?? localResolver?.resolve(current)
|
||||
: undefined,
|
||||
oidcProtocol: protocol,
|
||||
groupCatalog,
|
||||
}).inspect(request);
|
||||
if (!request.interactive || !report.ready) return report;
|
||||
if (current?.value.mode !== "oidc" || !protocol?.verifyDeviceFlow || !request.presentDeviceCode) {
|
||||
return {
|
||||
ready: false,
|
||||
mode: report.mode,
|
||||
checks: [{
|
||||
level: "error",
|
||||
code: "oidc_device_flow_unavailable",
|
||||
message: "Interactive authentication diagnostics require OIDC device authorization.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
try {
|
||||
const identity = await protocol.verifyDeviceFlow(
|
||||
request.signal ?? AbortSignal.timeout(10 * 60_000), request.presentDeviceCode,
|
||||
);
|
||||
// Exact names only: unrelated provider groups are neither emitted nor retained.
|
||||
const mappedRoles = new Set<string>();
|
||||
for (const [configuredGroup, roles] of Object.entries(current.value.authorization.groupRoles)) {
|
||||
if (!identity.groups.includes(configuredGroup)) continue;
|
||||
for (const role of roles) mappedRoles.add(role);
|
||||
}
|
||||
if (mappedRoles.size === 0) {
|
||||
return {
|
||||
ready: false,
|
||||
mode: "oidc",
|
||||
checks: [{
|
||||
level: "error",
|
||||
code: "oidc_groups_claim_invalid",
|
||||
message: "The OIDC device-flow identity could not be validated.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
return report;
|
||||
} catch (error) {
|
||||
return {
|
||||
ready: false,
|
||||
mode: "oidc",
|
||||
checks: [{
|
||||
level: "error",
|
||||
code: error instanceof OidcDeviceFlowUnavailableError
|
||||
? "oidc_device_flow_unavailable"
|
||||
: "oidc_groups_claim_invalid",
|
||||
message: error instanceof OidcDeviceFlowUnavailableError
|
||||
? "OIDC device authorization is unavailable."
|
||||
: "The OIDC device-flow identity could not be validated.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface DiagnosticCommandDependencies {
|
||||
diagnoser: AuthDiagnoser;
|
||||
secretValues?: readonly string[];
|
||||
stdout: (line: string) => void;
|
||||
stderr: (line: string) => void;
|
||||
}
|
||||
|
||||
function genericFailure(): AuthDiagnostics {
|
||||
return {
|
||||
ready: false,
|
||||
mode: "none",
|
||||
checks: [{ level: "error", code: "auth_config_invalid", message: "Authentication configuration is unavailable." }],
|
||||
};
|
||||
}
|
||||
|
||||
function redact(value: string, secrets: readonly string[]): string {
|
||||
let result = value;
|
||||
for (const secret of [...secrets].filter(Boolean).sort((left, right) => right.length - left.length)) {
|
||||
result = result.replaceAll(secret, "[REDACTED]");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function redactedReport(report: AuthDiagnostics, secrets: readonly string[]): AuthDiagnostics {
|
||||
return {
|
||||
...report,
|
||||
checks: report.checks.map((check): AuthDiagnostic => ({
|
||||
...check,
|
||||
message: redact(check.message, secrets),
|
||||
...(check.field === undefined ? {} : { field: redact(check.field, secrets) }),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function parseArguments(args: readonly string[]): { json: true; interactive: boolean } | undefined {
|
||||
let json = false;
|
||||
let interactive = false;
|
||||
for (const arg of args) {
|
||||
if (arg === "--json" && !json) json = true;
|
||||
else if (arg === "--interactive" && !interactive) interactive = true;
|
||||
else return undefined;
|
||||
}
|
||||
return json ? { json: true, interactive } : undefined;
|
||||
}
|
||||
|
||||
/** A bounded machine command: stdout receives exactly one final report and no progress text. */
|
||||
export async function runDiagnosticCommand(
|
||||
args: readonly string[],
|
||||
dependencies: DiagnosticCommandDependencies,
|
||||
): Promise<number> {
|
||||
const options = parseArguments(args);
|
||||
if (!options) {
|
||||
dependencies.stderr("usage: diagnostic-command.js --json [--interactive]");
|
||||
return 2;
|
||||
}
|
||||
let report: AuthDiagnostics;
|
||||
const secrets = dependencies.secretValues ?? [];
|
||||
try {
|
||||
report = await dependencies.diagnoser.inspect({
|
||||
live: true,
|
||||
...(options.interactive ? {
|
||||
interactive: true,
|
||||
presentDeviceCode: (uri: string, code: string) => dependencies.stderr(
|
||||
redact(`Open ${uri} and enter code ${code}`, secrets),
|
||||
),
|
||||
} : {}),
|
||||
});
|
||||
} catch {
|
||||
report = genericFailure();
|
||||
}
|
||||
const decoded = decodeAuthDiagnostics(report) ?? genericFailure();
|
||||
const safe = decodeAuthDiagnostics(redactedReport(decoded, secrets)) ?? genericFailure();
|
||||
dependencies.stdout(`${JSON.stringify(safe)}\n`);
|
||||
return safe.ready ? 0 : 1;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const exitCode = await runConfiguredDiagnosticCommand(
|
||||
process.argv.slice(2), process.env,
|
||||
(line) => process.stdout.write(line),
|
||||
(line) => process.stderr.write(`${line}\n`),
|
||||
);
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
|
||||
export async function runConfiguredDiagnosticCommand(
|
||||
args: readonly string[],
|
||||
env: Record<string, string | undefined>,
|
||||
stdout: (line: string) => void,
|
||||
stderr: (line: string) => void,
|
||||
): Promise<number> {
|
||||
let diagnoser: AuthDiagnoser = { inspect: async () => genericFailure() };
|
||||
let secretValues: readonly string[] | undefined;
|
||||
try {
|
||||
const config = loadConfig(env);
|
||||
// Complete this preflight before constructing a diagnoser that may forward a device prompt.
|
||||
secretValues = configuredSecretValues(config);
|
||||
diagnoser = createConfiguredAuthDiagnoser(config);
|
||||
} catch { /* turn startup or corpus faults into the closed report below */ }
|
||||
return runDiagnosticCommand(args, {
|
||||
diagnoser,
|
||||
...(secretValues === undefined ? {} : { secretValues }),
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
}
|
||||
|
||||
if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
void main();
|
||||
}
|
||||
@@ -1,219 +0,0 @@
|
||||
import type { AuthenticationConfigProvider, AuthMode } from "./types.js";
|
||||
import type { LocalUserRegistry } from "./local-registry.js";
|
||||
import { OidcIssuerMismatchError, OidcJwksUnavailableError, type OidcProtocol } from "./oidc-client.js";
|
||||
import {
|
||||
createPosixAuthStorageBridge,
|
||||
createWindowsAuthStorageBridge,
|
||||
type WindowsAuthStorageBridge,
|
||||
} from "./windows-auth-storage.js";
|
||||
import { isUsableAuthenticationSecret, type AuthenticationSecretReference } from "./secret-policy.js";
|
||||
import type { AuthDiagnostic, AuthDiagnosticCode, AuthDiagnostics, GroupCatalog } from "./group-catalog.js";
|
||||
|
||||
export type { AuthDiagnostic, AuthDiagnosticCode, AuthDiagnostics } from "./group-catalog.js";
|
||||
|
||||
const LIVE_DIAGNOSTIC_TIMEOUT_MS = 30_000;
|
||||
|
||||
export interface AuthDiagnoser {
|
||||
inspect(options: {
|
||||
live: boolean;
|
||||
interactive?: boolean;
|
||||
signal?: AbortSignal;
|
||||
/** Device-code presentation is transient operator output, never persisted diagnostic state. */
|
||||
presentDeviceCode?: (uri: string, code: string) => void;
|
||||
}): Promise<AuthDiagnostics>;
|
||||
}
|
||||
|
||||
export interface AuthDiagnoserDependencies {
|
||||
authMode: AuthMode;
|
||||
authStateRoot: string;
|
||||
/** Platform integrations may inject an equivalent side-effect-free owner/ACL validator. */
|
||||
sessionRootValidator?: (root: string) => void | Promise<void>;
|
||||
windowsStorageBridge?: Pick<WindowsAuthStorageBridge, "validateRoot">;
|
||||
posixStorageBridge?: Pick<WindowsAuthStorageBridge, "validateRoot">;
|
||||
authentication?: AuthenticationConfigProvider;
|
||||
secrets?: ReadonlyMap<string, string>;
|
||||
localUserRegistry?: LocalUserRegistry;
|
||||
oidcProtocol?: OidcProtocol;
|
||||
groupCatalog?: GroupCatalog;
|
||||
}
|
||||
|
||||
function check(code: AuthDiagnosticCode, message: string, field?: string): AuthDiagnostic {
|
||||
return { level: "error", code, message, ...(field === undefined ? {} : { field }) };
|
||||
}
|
||||
|
||||
function ordered(checks: readonly AuthDiagnostic[]): readonly AuthDiagnostic[] {
|
||||
const unique = new Map<string, AuthDiagnostic>();
|
||||
for (const item of checks) unique.set(`${item.code}\u0000${item.field ?? ""}`, item);
|
||||
return [...unique.values()].sort((left, right) => {
|
||||
const leftKey = `${left.code}\u0000${left.field ?? ""}`;
|
||||
const rightKey = `${right.code}\u0000${right.field ?? ""}`;
|
||||
return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0;
|
||||
});
|
||||
}
|
||||
|
||||
function stableCompare(left: string, right: string): number {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function abortReason(signal: AbortSignal): unknown {
|
||||
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
||||
}
|
||||
|
||||
function awaitWithAbort<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const abort = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(abortReason(signal));
|
||||
};
|
||||
if (signal.aborted) abort();
|
||||
else signal.addEventListener("abort", abort, { once: true });
|
||||
operation.then(
|
||||
(value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function startBeforeAbort<T>(signal: AbortSignal, operation: () => Promise<T>): Promise<T> {
|
||||
return Promise.resolve().then(() => {
|
||||
if (signal.aborted) throw abortReason(signal);
|
||||
return operation();
|
||||
});
|
||||
}
|
||||
|
||||
async function localRegistryIsUsable(deps: AuthDiagnoserDependencies): Promise<AuthDiagnostic | undefined> {
|
||||
try {
|
||||
if (!deps.localUserRegistry) {
|
||||
return check("local_user_registry_invalid", "The local user registry is unavailable.");
|
||||
}
|
||||
if (!await deps.localUserRegistry.hasEnabledAdmin()) {
|
||||
return check("local_admin_missing", "No enabled local administrator is configured.");
|
||||
}
|
||||
return undefined;
|
||||
} catch {
|
||||
return check("local_user_registry_invalid", "The local user registry is invalid.");
|
||||
}
|
||||
}
|
||||
|
||||
export function createAuthDiagnoser(deps: AuthDiagnoserDependencies): AuthDiagnoser {
|
||||
const validateSessionRoot = deps.sessionRootValidator ?? (process.platform === "win32"
|
||||
? (root: string) => (deps.windowsStorageBridge ?? createWindowsAuthStorageBridge()).validateRoot(root)
|
||||
: (root: string) => (deps.posixStorageBridge ?? createPosixAuthStorageBridge()).validateRoot(root));
|
||||
return {
|
||||
async inspect(options): Promise<AuthDiagnostics> {
|
||||
const checks: AuthDiagnostic[] = [];
|
||||
const signal = options.signal ?? new AbortController().signal;
|
||||
try {
|
||||
await validateSessionRoot(deps.authStateRoot);
|
||||
} catch {
|
||||
checks.push(check("auth_session_store_invalid", "The authentication session store is invalid."));
|
||||
}
|
||||
|
||||
if (deps.authMode === "none" || deps.authMode === "mock") {
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: deps.authMode, checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: deps.authMode, checks: result };
|
||||
}
|
||||
if (deps.authMode === "upstream") {
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? {
|
||||
ready: true,
|
||||
mode: "upstream",
|
||||
checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }],
|
||||
}
|
||||
: { ready: false, mode: "upstream", checks: result };
|
||||
}
|
||||
|
||||
let loaded;
|
||||
try {
|
||||
if (!deps.authentication) throw new Error("missing authentication configuration");
|
||||
loaded = deps.authentication.current();
|
||||
} catch {
|
||||
checks.push(check(deps.authentication ? "auth_config_invalid" : "auth_config_incomplete", "Authentication configuration is unavailable."));
|
||||
return { ready: false, mode: deps.authMode, checks: ordered(checks) };
|
||||
}
|
||||
if (loaded.value.mode !== deps.authMode) {
|
||||
checks.push(check("auth_config_invalid", "Authentication mode does not match its configuration."));
|
||||
return { ready: false, mode: deps.authMode, checks: ordered(checks) };
|
||||
}
|
||||
|
||||
if (loaded.value.mode === "local") {
|
||||
const local = await localRegistryIsUsable(deps);
|
||||
if (local) checks.push(local);
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: "local", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: "local", checks: result };
|
||||
}
|
||||
|
||||
const requiredSecrets: readonly AuthenticationSecretReference[] = ["THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN"];
|
||||
if (requiredSecrets.some((name) => !isUsableAuthenticationSecret(name, deps.secrets?.get(name)))) {
|
||||
checks.push(check("oidc_secret_missing", "A required OIDC or group catalog secret is unavailable."));
|
||||
}
|
||||
if (!options.live || checks.length > 0) {
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: "oidc", checks: result };
|
||||
}
|
||||
const mappedGroupNames = Object.keys(loaded.value.authorization.groupRoles).sort(stableCompare);
|
||||
|
||||
const deadline = new AbortController();
|
||||
const deadlineTimer = setTimeout(() => deadline.abort(), LIVE_DIAGNOSTIC_TIMEOUT_MS);
|
||||
deadlineTimer.unref();
|
||||
const liveSignal = AbortSignal.any([signal, deadline.signal]);
|
||||
try {
|
||||
if (!deps.oidcProtocol) {
|
||||
checks.push(check("oidc_discovery_unreachable", "The OIDC provider is unavailable."));
|
||||
} else {
|
||||
try {
|
||||
await awaitWithAbort(startBeforeAbort(liveSignal, () => deps.oidcProtocol!.diagnose(liveSignal)), liveSignal);
|
||||
} catch (error) {
|
||||
checks.push(check(
|
||||
error instanceof OidcIssuerMismatchError
|
||||
? "oidc_issuer_mismatch"
|
||||
: error instanceof OidcJwksUnavailableError
|
||||
? "oidc_jwks_unreachable"
|
||||
: "oidc_discovery_unreachable",
|
||||
"The OIDC provider could not be validated.",
|
||||
));
|
||||
}
|
||||
}
|
||||
if (!liveSignal.aborted) {
|
||||
if (!deps.groupCatalog) {
|
||||
checks.push(check("oidc_group_catalog_unreachable", "The configured group catalog cannot be certified."));
|
||||
} else {
|
||||
try {
|
||||
checks.push(...await awaitWithAbort(startBeforeAbort(liveSignal, () => deps.groupCatalog!.verifyConfiguredGroups(
|
||||
mappedGroupNames, liveSignal,
|
||||
)), liveSignal));
|
||||
} catch {
|
||||
checks.push(check("oidc_group_catalog_unreachable", "The configured group catalog is unavailable."));
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(deadlineTimer);
|
||||
}
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: "oidc", checks: result };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
/** The fixed machine contract shared by the Authentik catalog and auth diagnostics. */
|
||||
export type AuthDiagnosticCode =
|
||||
| "auth_ready"
|
||||
| "auth_config_incomplete"
|
||||
| "auth_config_invalid"
|
||||
| "auth_session_store_invalid"
|
||||
| "local_user_registry_invalid"
|
||||
| "local_admin_missing"
|
||||
| "oidc_secret_missing"
|
||||
| "oidc_discovery_unreachable"
|
||||
| "oidc_issuer_mismatch"
|
||||
| "oidc_jwks_unreachable"
|
||||
| "oidc_group_catalog_unreachable"
|
||||
| "oidc_group_catalog_unauthorized"
|
||||
| "oidc_mapped_group_missing"
|
||||
| "oidc_mapped_group_ambiguous"
|
||||
| "oidc_groups_claim_invalid"
|
||||
| "oidc_device_flow_unavailable";
|
||||
|
||||
export interface AuthDiagnostic {
|
||||
level: "error" | "info";
|
||||
code: AuthDiagnosticCode;
|
||||
message: string;
|
||||
field?: string;
|
||||
}
|
||||
|
||||
export interface AuthDiagnostics {
|
||||
ready: boolean;
|
||||
mode: "local" | "oidc" | "upstream" | "none" | "mock";
|
||||
checks: readonly AuthDiagnostic[];
|
||||
}
|
||||
|
||||
const diagnosticCodes = new Set<AuthDiagnosticCode>([
|
||||
"auth_ready", "auth_config_incomplete", "auth_config_invalid", "auth_session_store_invalid",
|
||||
"local_user_registry_invalid", "local_admin_missing", "oidc_secret_missing",
|
||||
"oidc_discovery_unreachable", "oidc_issuer_mismatch", "oidc_jwks_unreachable",
|
||||
"oidc_group_catalog_unreachable", "oidc_group_catalog_unauthorized", "oidc_mapped_group_missing",
|
||||
"oidc_mapped_group_ambiguous", "oidc_groups_claim_invalid", "oidc_device_flow_unavailable",
|
||||
]);
|
||||
const diagnosticModes = new Set<AuthDiagnostics["mode"]>(["local", "oidc", "upstream", "none", "mock"]);
|
||||
const fieldCodes = new Set<AuthDiagnosticCode>(["oidc_mapped_group_missing", "oidc_mapped_group_ambiguous"]);
|
||||
|
||||
function exactObject(value: unknown, keys: readonly string[]): Record<string, unknown> | undefined {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||
const source = value as Record<string, unknown>;
|
||||
const actual = Object.keys(source);
|
||||
return actual.length === keys.length && actual.every((key) => keys.includes(key)) ? source : undefined;
|
||||
}
|
||||
|
||||
function safeText(value: unknown): value is string {
|
||||
return typeof value === "string" && value.length > 0 && value.length <= 512
|
||||
&& value.trim() === value && !/\p{Cc}/u.test(value);
|
||||
}
|
||||
|
||||
/** Strict decoder for the machine contract shared with tht and the frontend. */
|
||||
export function decodeAuthDiagnostics(value: unknown): AuthDiagnostics | undefined {
|
||||
const source = exactObject(value, ["ready", "mode", "checks"]);
|
||||
if (!source || typeof source.ready !== "boolean" || typeof source.mode !== "string"
|
||||
|| !diagnosticModes.has(source.mode as AuthDiagnostics["mode"])
|
||||
|| !Array.isArray(source.checks) || source.checks.length === 0 || source.checks.length > 129) return undefined;
|
||||
const seen = new Set<string>();
|
||||
const checks: AuthDiagnostic[] = [];
|
||||
for (const value of source.checks) {
|
||||
const raw = value && typeof value === "object" && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: undefined;
|
||||
const check = raw && exactObject(raw, raw.field === undefined
|
||||
? ["level", "code", "message"]
|
||||
: ["level", "code", "message", "field"]);
|
||||
if (!check || (check.level !== "error" && check.level !== "info")
|
||||
|| typeof check.code !== "string" || !diagnosticCodes.has(check.code as AuthDiagnosticCode)
|
||||
|| !safeText(check.message) || (check.field !== undefined && !safeText(check.field))) return undefined;
|
||||
const code = check.code as AuthDiagnosticCode;
|
||||
if (check.field !== undefined && !fieldCodes.has(code)) return undefined;
|
||||
const key = `${code}\u0000${check.field ?? ""}`;
|
||||
if (seen.has(key)) return undefined;
|
||||
seen.add(key);
|
||||
checks.push({
|
||||
level: check.level,
|
||||
code,
|
||||
message: check.message,
|
||||
...(check.field === undefined ? {} : { field: check.field }),
|
||||
});
|
||||
}
|
||||
if (source.ready) {
|
||||
if (checks.length !== 1 || checks[0].level !== "info" || checks[0].code !== "auth_ready"
|
||||
|| checks[0].field !== undefined) return undefined;
|
||||
} else if (!checks.some(({ level }) => level === "error")
|
||||
|| checks.some(({ code }) => code === "auth_ready")) return undefined;
|
||||
return { ready: source.ready, mode: source.mode as AuthDiagnostics["mode"], checks };
|
||||
}
|
||||
|
||||
/** A provider-specific proof that only the configured authorization groups exist. */
|
||||
export interface GroupCatalog {
|
||||
verifyConfiguredGroups(names: readonly string[], signal: AbortSignal): Promise<readonly AuthDiagnostic[]>;
|
||||
}
|
||||
@@ -1,329 +0,0 @@
|
||||
import {
|
||||
closeSync,
|
||||
constants,
|
||||
fstatSync,
|
||||
lstatSync,
|
||||
openSync,
|
||||
readSync,
|
||||
realpathSync,
|
||||
} from "node:fs";
|
||||
import type { Stats } from "node:fs";
|
||||
import { dirname, isAbsolute, join, normalize } from "node:path";
|
||||
import { parseDocument } from "yaml";
|
||||
import { z } from "zod";
|
||||
import { isValidPasswordHash, verifyPassword, verifyWithDummy } from "./password.js";
|
||||
import type { LoadedAuthConfig, LocalUserRecord, Role } from "./types.js";
|
||||
import { createWindowsAuthStorageBridge, type WindowsAuthStorageBridge } from "./windows-auth-storage.js";
|
||||
|
||||
const MAX_USERS_YAML_BYTES = 1 << 20;
|
||||
const USERNAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._@-]{2,63}$/;
|
||||
const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
const invalid = (): Error => new Error("local_user_registry_invalid");
|
||||
|
||||
function runtimeOwner(): number {
|
||||
if (process.platform === "win32" || typeof process.geteuid !== "function") throw invalid();
|
||||
const owner = process.geteuid();
|
||||
if (!Number.isSafeInteger(owner) || owner < 0) throw invalid();
|
||||
return owner;
|
||||
}
|
||||
|
||||
export type { LocalUserRecord } from "./types.js";
|
||||
|
||||
export interface LocalUserRegistry {
|
||||
/** Safe production diagnostic probe; never returns user records or hashes. */
|
||||
hasEnabledAdmin(): Promise<boolean>;
|
||||
findByUsername(username: string): Promise<LocalUserRecord | undefined>;
|
||||
findBySubject(id: string): Promise<LocalUserRecord | undefined>;
|
||||
verify(user: LocalUserRecord | undefined, password: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
/** Keeps only the registry named by the current coherent authentication-config snapshot. */
|
||||
export interface CurrentLocalUserRegistryResolver {
|
||||
resolve(loaded: LoadedAuthConfig): LocalUserRegistry | undefined;
|
||||
}
|
||||
|
||||
/** Native Windows obtains protected registry bytes only from the hidden tht bridge. */
|
||||
export interface LocalUserRegistryOptions {
|
||||
windowsStorageBridge?: Pick<WindowsAuthStorageBridge, "readLocalUsers">;
|
||||
}
|
||||
|
||||
interface FileIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
}
|
||||
|
||||
interface DirectoryIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
mode: number;
|
||||
}
|
||||
|
||||
interface RegistryIdentity {
|
||||
file: FileIdentity;
|
||||
directory: DirectoryIdentity;
|
||||
}
|
||||
|
||||
const roleSchema = z.enum(ROLES);
|
||||
const userSchema = z.strictObject({
|
||||
id: z.string().regex(UUID_V4_PATTERN),
|
||||
username: z.string().regex(USERNAME_PATTERN),
|
||||
displayName: z.string().optional().refine((value) => value === undefined || !/\p{Cc}/u.test(value)),
|
||||
passwordHash: z.string().refine(isValidPasswordHash),
|
||||
roles: z.array(roleSchema).min(1).superRefine((roles, context) => {
|
||||
if (new Set(roles).size !== roles.length) context.addIssue({ code: "custom", message: "duplicate role" });
|
||||
}),
|
||||
enabled: z.boolean(),
|
||||
authRevision: z.number().int().positive().safe(),
|
||||
});
|
||||
const registrySchema = z.strictObject({ version: z.literal(1), users: z.array(userSchema).min(1) });
|
||||
|
||||
function normalizeUsername(username: string): string {
|
||||
return username.replace(/[A-Z]/g, (character) => character.toLowerCase());
|
||||
}
|
||||
|
||||
function sameFileIdentity(left: FileIdentity, right: FileIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid
|
||||
&& left.size === right.size && left.mtimeMs === right.mtimeMs;
|
||||
}
|
||||
|
||||
function sameDirectoryIdentity(left: DirectoryIdentity, right: DirectoryIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid && left.mode === right.mode;
|
||||
}
|
||||
|
||||
function sameIdentity(left: RegistryIdentity, right: RegistryIdentity): boolean {
|
||||
return sameFileIdentity(left.file, right.file) && sameDirectoryIdentity(left.directory, right.directory);
|
||||
}
|
||||
|
||||
function validateCanonicalPath(path: string): void {
|
||||
if (typeof path !== "string" || path.length === 0 || path.includes("\0") || !isAbsolute(path) || normalize(path) !== path) throw invalid();
|
||||
const parent = dirname(path);
|
||||
if (realpathSync(parent) !== parent) throw invalid();
|
||||
}
|
||||
|
||||
function fileMetadata(info: Stats, owner: number): FileIdentity {
|
||||
if (!info.isFile() || info.uid !== owner || info.nlink !== 1 || (info.mode & 0o7777) !== 0o600) throw invalid();
|
||||
if (info.size < 0 || info.size > MAX_USERS_YAML_BYTES) throw invalid();
|
||||
return { dev: info.dev, ino: info.ino, uid: info.uid, size: info.size, mtimeMs: info.mtimeMs };
|
||||
}
|
||||
|
||||
function directoryMetadata(info: Stats, owner: number): DirectoryIdentity {
|
||||
if (!info.isDirectory() || info.uid !== owner || (info.mode & 0o7777) !== 0o700) throw invalid();
|
||||
return { dev: info.dev, ino: info.ino, uid: info.uid, mode: info.mode & 0o7777 };
|
||||
}
|
||||
|
||||
function directoryIdentity(path: string, owner: number): DirectoryIdentity {
|
||||
const parent = dirname(path);
|
||||
if (realpathSync(parent) !== parent) throw invalid();
|
||||
return directoryMetadata(lstatSync(parent) as Stats, owner);
|
||||
}
|
||||
|
||||
function registryIdentity(path: string, owner: number): RegistryIdentity {
|
||||
validateCanonicalPath(path);
|
||||
const info = lstatSync(path);
|
||||
return { file: fileMetadata(info as Stats, owner), directory: directoryIdentity(path, owner) };
|
||||
}
|
||||
|
||||
function openDirectoryDescriptor(path: string): number | undefined {
|
||||
if (process.platform === "win32") return undefined;
|
||||
const flags = constants.O_RDONLY
|
||||
| (constants.O_DIRECTORY ?? 0)
|
||||
| (constants.O_NOFOLLOW ?? 0)
|
||||
| (constants.O_NONBLOCK ?? 0);
|
||||
return openSync(path, flags);
|
||||
}
|
||||
|
||||
function readBounded(path: string, owner: number): { source: string; identity: RegistryIdentity } {
|
||||
validateCanonicalPath(path);
|
||||
const beforeDirectory = directoryIdentity(path, owner);
|
||||
const beforePath = lstatSync(path);
|
||||
const before = fileMetadata(beforePath as Stats, owner);
|
||||
let directoryDescriptor: number | undefined;
|
||||
let descriptor: number | undefined;
|
||||
try {
|
||||
directoryDescriptor = openDirectoryDescriptor(dirname(path));
|
||||
const openedDirectory = directoryDescriptor === undefined
|
||||
? beforeDirectory
|
||||
: directoryMetadata(fstatSync(directoryDescriptor) as Stats, owner);
|
||||
if (!sameDirectoryIdentity(beforeDirectory, openedDirectory)) throw invalid();
|
||||
descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
const opened = fileMetadata(fstatSync(descriptor) as Stats, owner);
|
||||
if (!sameFileIdentity(before, opened)) throw invalid();
|
||||
const buffer = Buffer.allocUnsafe(MAX_USERS_YAML_BYTES + 1);
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const bytesRead = readSync(descriptor, buffer, offset, buffer.length - offset, null);
|
||||
if (bytesRead === 0) break;
|
||||
offset += bytesRead;
|
||||
}
|
||||
if (offset > MAX_USERS_YAML_BYTES) throw invalid();
|
||||
const after = fileMetadata(fstatSync(descriptor) as Stats, owner);
|
||||
const afterPath = fileMetadata(lstatSync(path) as Stats, owner);
|
||||
const afterDirectory = directoryMetadata(lstatSync(dirname(path)) as Stats, owner);
|
||||
const afterOpenedDirectory = directoryDescriptor === undefined
|
||||
? afterDirectory
|
||||
: directoryMetadata(fstatSync(directoryDescriptor) as Stats, owner);
|
||||
if (!sameFileIdentity(opened, after) || !sameFileIdentity(after, afterPath)
|
||||
|| !sameDirectoryIdentity(beforeDirectory, afterDirectory)
|
||||
|| !sameDirectoryIdentity(openedDirectory, afterOpenedDirectory)) throw invalid();
|
||||
const source = new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, offset));
|
||||
return { source, identity: { file: after, directory: afterDirectory } };
|
||||
} catch {
|
||||
throw invalid();
|
||||
} finally {
|
||||
if (descriptor !== undefined) {
|
||||
try { closeSync(descriptor); } catch { /* sanitized by design */ }
|
||||
}
|
||||
if (directoryDescriptor !== undefined) {
|
||||
try { closeSync(directoryDescriptor); } catch { /* sanitized by design */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function parseLocalUserRegistrySource(source: string): readonly LocalUserRecord[] {
|
||||
try {
|
||||
const document = parseDocument(source, { uniqueKeys: true });
|
||||
if (document.errors.length > 0 || document.warnings.length > 0) throw invalid();
|
||||
const parsed = registrySchema.parse(document.toJSON());
|
||||
const ids = new Set<string>();
|
||||
const usernames = new Set<string>();
|
||||
const records = parsed.users.map((user) => {
|
||||
const normalizedUsername = normalizeUsername(user.username);
|
||||
if (ids.has(user.id) || usernames.has(normalizedUsername)) throw invalid();
|
||||
ids.add(user.id);
|
||||
usernames.add(normalizedUsername);
|
||||
return Object.freeze({
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
normalizedUsername,
|
||||
...(user.displayName === undefined ? {} : { displayName: user.displayName }),
|
||||
passwordHash: user.passwordHash,
|
||||
roles: Object.freeze([...user.roles]) as readonly Role[],
|
||||
enabled: user.enabled,
|
||||
authRevision: user.authRevision,
|
||||
});
|
||||
});
|
||||
return Object.freeze(records);
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function load(path: string, owner: number): { records: readonly LocalUserRecord[]; identity: RegistryIdentity } {
|
||||
const read = readBounded(path, owner);
|
||||
return { records: parseLocalUserRegistrySource(read.source), identity: read.identity };
|
||||
}
|
||||
|
||||
export function createLocalUserRegistry(usersPath: string, options: LocalUserRegistryOptions = {}): LocalUserRegistry {
|
||||
let cached: { records: readonly LocalUserRecord[]; identity: RegistryIdentity } | undefined;
|
||||
const windowsStorage = process.platform === "win32"
|
||||
? options.windowsStorageBridge ?? createWindowsAuthStorageBridge()
|
||||
: undefined;
|
||||
|
||||
function currentPosix(): readonly LocalUserRecord[] {
|
||||
try {
|
||||
const owner = runtimeOwner();
|
||||
const before = registryIdentity(usersPath, owner);
|
||||
if (cached && sameIdentity(cached.identity, before)) return cached.records;
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
const loaded = load(usersPath, owner);
|
||||
if (sameIdentity(loaded.identity, registryIdentity(usersPath, owner))) {
|
||||
cached = loaded;
|
||||
return loaded.records;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
throw invalid();
|
||||
}
|
||||
|
||||
async function current(): Promise<readonly LocalUserRecord[]> {
|
||||
if (process.platform !== "win32") return currentPosix();
|
||||
try {
|
||||
if (!windowsStorage) throw invalid();
|
||||
const contents = await windowsStorage.readLocalUsers(usersPath);
|
||||
if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > MAX_USERS_YAML_BYTES) throw invalid();
|
||||
return parseLocalUserRegistrySource(new TextDecoder("utf-8", { fatal: true }).decode(contents));
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
async function operationalRecords(): Promise<readonly LocalUserRecord[]> {
|
||||
const records = await current();
|
||||
if (!records.some((user) => user.enabled && user.roles.includes("admin"))) throw invalid();
|
||||
return records;
|
||||
}
|
||||
|
||||
return {
|
||||
async hasEnabledAdmin(): Promise<boolean> {
|
||||
return (await current()).some((user) => user.enabled && user.roles.includes("admin"));
|
||||
},
|
||||
async findByUsername(username: string): Promise<LocalUserRecord | undefined> {
|
||||
const normalized = normalizeUsername(username);
|
||||
return (await operationalRecords()).find((user) => user.normalizedUsername === normalized);
|
||||
},
|
||||
async findBySubject(id: string): Promise<LocalUserRecord | undefined> {
|
||||
return (await operationalRecords()).find((user) => user.id === id);
|
||||
},
|
||||
async verify(user: LocalUserRecord | undefined, password: string): Promise<boolean> {
|
||||
if (!user || !user.enabled) {
|
||||
await verifyWithDummy(password);
|
||||
return false;
|
||||
}
|
||||
return await verifyPassword(password, user.passwordHash);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createCurrentLocalUserRegistryResolver(options: LocalUserRegistryOptions = {}): CurrentLocalUserRegistryResolver {
|
||||
let current: { key: object | string; registry: LocalUserRegistry } | undefined;
|
||||
return {
|
||||
resolve(loaded: LoadedAuthConfig): LocalUserRegistry | undefined {
|
||||
if (loaded.value.mode !== "local") return undefined;
|
||||
const runtimeProjection = loaded.runtimeProjection;
|
||||
const projectedUsers = runtimeProjection?.localUsers;
|
||||
if (projectedUsers && runtimeProjection) {
|
||||
if (current && current.key === runtimeProjection) return current.registry;
|
||||
const registry = createInMemoryLocalUserRegistry(projectedUsers);
|
||||
current = { key: runtimeProjection, registry };
|
||||
return registry;
|
||||
}
|
||||
const usersPath = join(dirname(loaded.sourcePath), loaded.value.local.usersFile);
|
||||
if (current && current.key === usersPath) return current.registry;
|
||||
const registry = createLocalUserRegistry(usersPath, options);
|
||||
current = { key: usersPath, registry };
|
||||
return registry;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createInMemoryLocalUserRegistry(records: readonly LocalUserRecord[]): LocalUserRegistry {
|
||||
async function operationalRecords(): Promise<readonly LocalUserRecord[]> {
|
||||
if (!records.some((user) => user.enabled && user.roles.includes("admin"))) throw invalid();
|
||||
return records;
|
||||
}
|
||||
return {
|
||||
async hasEnabledAdmin(): Promise<boolean> {
|
||||
return records.some((user) => user.enabled && user.roles.includes("admin"));
|
||||
},
|
||||
async findByUsername(username: string): Promise<LocalUserRecord | undefined> {
|
||||
return (await operationalRecords()).find((user) => user.normalizedUsername === normalizeUsername(username));
|
||||
},
|
||||
async findBySubject(id: string): Promise<LocalUserRecord | undefined> {
|
||||
return (await operationalRecords()).find((user) => user.id === id);
|
||||
},
|
||||
async verify(user: LocalUserRecord | undefined, password: string): Promise<boolean> {
|
||||
if (!user || !user.enabled) {
|
||||
await verifyWithDummy(password);
|
||||
return false;
|
||||
}
|
||||
return await verifyPassword(password, user.passwordHash);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,658 +0,0 @@
|
||||
import {
|
||||
authorizationCodeGrant,
|
||||
buildAuthorizationUrl,
|
||||
calculatePKCECodeChallenge,
|
||||
customFetch,
|
||||
discovery,
|
||||
initiateDeviceAuthorization,
|
||||
pollDeviceAuthorizationGrant,
|
||||
type Configuration,
|
||||
type CustomFetch,
|
||||
} from "openid-client";
|
||||
import { constants, createPublicKey, verify as verifySignature } from "node:crypto";
|
||||
import { parseConfiguredTransportUrl } from "./url-policy.js";
|
||||
import { isUsableAuthenticationSecret } from "./secret-policy.js";
|
||||
|
||||
export interface OidcIdentity {
|
||||
issuer: string;
|
||||
subject: string;
|
||||
displayName?: string;
|
||||
groups: readonly string[];
|
||||
tokenExpiresAt: Date;
|
||||
}
|
||||
|
||||
export interface OidcProtocol {
|
||||
authorizationUrl(input: { state: string; nonce: string; codeVerifier: string }): Promise<URL>;
|
||||
callback(input: { currentUrl: URL; state: string; nonce: string; codeVerifier: string }): Promise<OidcIdentity>;
|
||||
diagnose(signal: AbortSignal): Promise<void>;
|
||||
verifyDeviceFlow?(signal: AbortSignal, present: (uri: string, code: string) => void): Promise<OidcIdentity>;
|
||||
}
|
||||
|
||||
export class OidcProtocolError extends Error {
|
||||
constructor(message = "oidc_protocol_invalid") {
|
||||
super(message);
|
||||
this.name = "OidcProtocolError";
|
||||
}
|
||||
}
|
||||
|
||||
/** A safe operational distinction for callers and diagnostics; provider details never cross this boundary. */
|
||||
export class OidcProviderUnavailableError extends OidcProtocolError {
|
||||
constructor() {
|
||||
super("oidc_provider_unavailable");
|
||||
this.name = "OidcProviderUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
/** The discovery document resolved, but its signed-token key set could not be certified. */
|
||||
export class OidcJwksUnavailableError extends OidcProtocolError {
|
||||
constructor() {
|
||||
super("oidc_jwks_unreachable");
|
||||
this.name = "OidcJwksUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Discovery completed with metadata for a different issuer than the configured trust anchor. */
|
||||
export class OidcIssuerMismatchError extends OidcProtocolError {
|
||||
constructor() {
|
||||
super("oidc_issuer_mismatch");
|
||||
this.name = "OidcIssuerMismatchError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Device authorization is optional OIDC metadata and must never fall back to a browser flow. */
|
||||
export class OidcDeviceFlowUnavailableError extends OidcProtocolError {
|
||||
constructor() {
|
||||
super("oidc_device_flow_unavailable");
|
||||
this.name = "OidcDeviceFlowUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface OidcProtocolOptions {
|
||||
issuer: string;
|
||||
clientId: string;
|
||||
clientSecret: string;
|
||||
callbackUrl: string;
|
||||
scopes: readonly string[];
|
||||
groupsClaim: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
httpTimeoutMs?: number;
|
||||
jwksTimeoutMs?: number;
|
||||
}
|
||||
|
||||
const MAX_GROUPS = 128;
|
||||
const MAX_GROUP_LENGTH = 256;
|
||||
const MAX_ID_TOKEN_LENGTH = 16 * 1024;
|
||||
const MAX_OIDC_RESPONSE_BYTES = 1024 * 1024;
|
||||
const DEFAULT_HTTP_TIMEOUT_MS = 5_000;
|
||||
const MAX_HTTP_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_JWKS_TIMEOUT_MS = 5_000;
|
||||
const MAX_JWKS_TIMEOUT_MS = 30_000;
|
||||
const MAX_DEVICE_FLOW_TIMEOUT_MS = 10 * 60_000;
|
||||
const text = (value: unknown, maximum = 2048): value is string =>
|
||||
typeof value === "string" && value.length > 0 && value.length <= maximum && !/\p{Cc}/u.test(value);
|
||||
|
||||
function discoveryStringList(value: unknown): value is readonly string[] {
|
||||
return Array.isArray(value) && value.length > 0 && value.length <= 128
|
||||
&& value.every((item) => text(item, 128));
|
||||
}
|
||||
|
||||
function schemaValidDiscoveryMetadata(value: unknown): value is Record<string, unknown> & { issuer: string } {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
||||
const metadata = value as Record<string, unknown>;
|
||||
const issuer = metadata.issuer;
|
||||
const authorizationEndpoint = metadata.authorization_endpoint;
|
||||
const tokenEndpoint = metadata.token_endpoint;
|
||||
const jwksUri = metadata.jwks_uri;
|
||||
if (!text(issuer, 2048) || !text(authorizationEndpoint, 2048)
|
||||
|| !text(tokenEndpoint, 2048) || !text(jwksUri, 2048)
|
||||
|| !discoveryStringList(metadata.response_types_supported)
|
||||
|| !discoveryStringList(metadata.subject_types_supported)
|
||||
|| !discoveryStringList(metadata.id_token_signing_alg_values_supported)) return false;
|
||||
try {
|
||||
configuredHttpsUrl(issuer);
|
||||
httpsEndpoint(authorizationEndpoint);
|
||||
httpsEndpoint(tokenEndpoint);
|
||||
httpsEndpoint(jwksUri);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function configuredHttpsUrl(value: string): URL {
|
||||
const url = parseConfiguredTransportUrl(value, { allowLoopbackHttp: false });
|
||||
if (!url) throw new OidcProtocolError();
|
||||
return url;
|
||||
}
|
||||
|
||||
function configuredCallbackUrl(value: string): URL {
|
||||
const url = parseConfiguredTransportUrl(value, { allowLoopbackHttp: true });
|
||||
if (!url) throw new OidcProtocolError();
|
||||
return url;
|
||||
}
|
||||
|
||||
function httpsEndpoint(value: unknown): URL {
|
||||
if (!text(value, 2048)) throw new OidcProtocolError();
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(value);
|
||||
} catch {
|
||||
throw new OidcProtocolError();
|
||||
}
|
||||
if (url.protocol !== "https:" || url.username || url.password || url.hash) throw new OidcProtocolError();
|
||||
return url;
|
||||
}
|
||||
|
||||
function groupsFromClaims(claims: Record<string, unknown>, name: string): string[] {
|
||||
const indirect = claims._claim_names;
|
||||
if ((indirect && typeof indirect === "object" && !Array.isArray(indirect)
|
||||
&& Object.prototype.hasOwnProperty.call(indirect, name))
|
||||
|| claims.hasgroups === true) throw new OidcProtocolError();
|
||||
const raw = claims[name];
|
||||
if (!Array.isArray(raw) || raw.length === 0 || raw.length > MAX_GROUPS) throw new OidcProtocolError();
|
||||
const groups: string[] = [];
|
||||
const unique = new Set<string>();
|
||||
for (const group of raw) {
|
||||
if (!text(group, MAX_GROUP_LENGTH) || group.trim().length === 0 || unique.has(group)) throw new OidcProtocolError();
|
||||
unique.add(group);
|
||||
groups.push(group);
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
function identityFromClaims(claims: Record<string, unknown>, options: OidcProtocolOptions): OidcIdentity {
|
||||
if (claims.iss !== options.issuer || !text(claims.sub, 512)) throw new OidcProtocolError();
|
||||
const audience = claims.aud;
|
||||
if (!(audience === options.clientId || (Array.isArray(audience) && audience.includes(options.clientId)))) {
|
||||
throw new OidcProtocolError();
|
||||
}
|
||||
if (typeof claims.exp !== "number" || !Number.isSafeInteger(claims.exp) || claims.exp * 1000 <= Date.now()) {
|
||||
throw new OidcProtocolError();
|
||||
}
|
||||
const tokenExpiresAt = new Date(claims.exp * 1000);
|
||||
if (Number.isNaN(tokenExpiresAt.getTime())) throw new OidcProtocolError();
|
||||
return {
|
||||
issuer: options.issuer,
|
||||
subject: claims.sub,
|
||||
...(text(claims.name, 256) ? { displayName: claims.name } : {}),
|
||||
groups: groupsFromClaims(claims, options.groupsClaim),
|
||||
tokenExpiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
function jsonPart(part: string): Record<string, unknown> {
|
||||
if (!/^[A-Za-z0-9_-]+$/.test(part) || part.length > MAX_ID_TOKEN_LENGTH) throw new OidcProtocolError();
|
||||
try {
|
||||
const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.from(part, "base64url")));
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new OidcProtocolError();
|
||||
return value as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new OidcProtocolError();
|
||||
}
|
||||
}
|
||||
|
||||
function signatureAlgorithm(algorithm: string): {
|
||||
digest: string | null;
|
||||
pss?: boolean;
|
||||
saltLength?: number;
|
||||
ecdsaPartLength?: number;
|
||||
} {
|
||||
switch (algorithm) {
|
||||
case "RS256": return { digest: "RSA-SHA256" };
|
||||
case "RS384": return { digest: "RSA-SHA384" };
|
||||
case "RS512": return { digest: "RSA-SHA512" };
|
||||
case "PS256": return { digest: "sha256", pss: true, saltLength: 32 };
|
||||
case "PS384": return { digest: "sha384", pss: true, saltLength: 48 };
|
||||
case "PS512": return { digest: "sha512", pss: true, saltLength: 64 };
|
||||
case "ES256": return { digest: "sha256", ecdsaPartLength: 32 };
|
||||
case "ES384": return { digest: "sha384", ecdsaPartLength: 48 };
|
||||
case "ES512": return { digest: "sha512", ecdsaPartLength: 66 };
|
||||
case "EdDSA": return { digest: null };
|
||||
default: throw new OidcProtocolError();
|
||||
}
|
||||
}
|
||||
|
||||
function derLength(length: number): Buffer {
|
||||
if (length < 128) return Buffer.from([length]);
|
||||
if (length < 256) return Buffer.from([0x81, length]);
|
||||
throw new OidcProtocolError();
|
||||
}
|
||||
|
||||
function derInteger(raw: Buffer): Buffer {
|
||||
let start = 0;
|
||||
while (start < raw.length - 1 && raw[start] === 0) start += 1;
|
||||
let value = raw.subarray(start);
|
||||
if ((value[0] & 0x80) !== 0) value = Buffer.concat([Buffer.from([0]), value]);
|
||||
return Buffer.concat([Buffer.from([0x02]), derLength(value.length), value]);
|
||||
}
|
||||
|
||||
function joseEcdsaSignatureToDer(signature: Buffer, partLength: number): Buffer {
|
||||
if (signature.length !== partLength * 2) throw new OidcProtocolError();
|
||||
const sequence = Buffer.concat([
|
||||
derInteger(signature.subarray(0, partLength)),
|
||||
derInteger(signature.subarray(partLength)),
|
||||
]);
|
||||
return Buffer.concat([Buffer.from([0x30]), derLength(sequence.length), sequence]);
|
||||
}
|
||||
|
||||
class OidcTransportError extends Error {
|
||||
constructor(readonly availability: boolean) {
|
||||
super(availability ? "oidc_provider_unavailable" : "oidc_provider_response_invalid");
|
||||
this.name = "OidcTransportError";
|
||||
}
|
||||
}
|
||||
|
||||
interface BoundedOidcTransport {
|
||||
customFetch: CustomFetch;
|
||||
request(
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
options?: { timeoutMs?: number; requireSuccess?: boolean },
|
||||
): Promise<Response>;
|
||||
}
|
||||
|
||||
function cancelReaderBestEffort(reader: ReadableStreamDefaultReader<Uint8Array>): void {
|
||||
try {
|
||||
void Promise.resolve(reader.cancel()).catch(() => undefined);
|
||||
} catch {
|
||||
// Cancellation is advisory; the deadline and rejection remain authoritative.
|
||||
}
|
||||
}
|
||||
|
||||
function cancelResponseBestEffort(response: Response): void {
|
||||
if (!response.body) return;
|
||||
try {
|
||||
void Promise.resolve(response.body.cancel()).catch(() => undefined);
|
||||
} catch {
|
||||
// A late response is never allowed to turn an already-bounded request into an unhandled rejection.
|
||||
}
|
||||
}
|
||||
|
||||
function abortedReason(signal: AbortSignal): unknown {
|
||||
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
||||
}
|
||||
|
||||
async function awaitWithAbort<T>(
|
||||
operation: Promise<T>,
|
||||
signal: AbortSignal,
|
||||
onLateResolution?: (value: T) => void,
|
||||
): Promise<T> {
|
||||
return await new Promise<T>((resolve, reject) => {
|
||||
let settled = signal.aborted;
|
||||
const cleanup = () => signal.removeEventListener("abort", aborted);
|
||||
const aborted = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(abortedReason(signal));
|
||||
};
|
||||
if (signal.aborted) reject(abortedReason(signal));
|
||||
else signal.addEventListener("abort", aborted, { once: true });
|
||||
operation.then(
|
||||
(value) => {
|
||||
if (settled) {
|
||||
try { onLateResolution?.(value); } catch { /* best-effort late cleanup only */ }
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function providerRequestUrl(input: RequestInfo | URL): URL {
|
||||
const value = input instanceof URL ? input.href : input instanceof Request ? input.url : input;
|
||||
return httpsEndpoint(value);
|
||||
}
|
||||
|
||||
function declaredResponseLength(response: Response): number | undefined {
|
||||
const declared = response.headers.get("content-length");
|
||||
if (declared === null) return undefined;
|
||||
if (!/^\d+$/.test(declared)) throw new OidcTransportError(false);
|
||||
const length = Number(declared);
|
||||
if (!Number.isSafeInteger(length) || length > MAX_OIDC_RESPONSE_BYTES) throw new OidcTransportError(false);
|
||||
return length;
|
||||
}
|
||||
|
||||
function safeBufferedResponse(response: Response, body: Buffer): Response {
|
||||
const noBodyStatus = response.status === 204 || response.status === 205 || response.status === 304;
|
||||
// Node accepts Buffer as a fetch body; the DOM declaration in this project does not model it.
|
||||
return new Response(noBodyStatus ? null : body as unknown as BodyInit, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
}
|
||||
|
||||
async function boundedResponse(
|
||||
response: Response,
|
||||
signal: AbortSignal,
|
||||
requireSuccess: boolean,
|
||||
): Promise<Response> {
|
||||
const reader = response.body?.getReader();
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
let completed = false;
|
||||
try {
|
||||
if (response.redirected || response.type === "opaqueredirect" || response.status >= 300 && response.status < 400) {
|
||||
throw new OidcTransportError(false);
|
||||
}
|
||||
if (requireSuccess && !response.ok) throw new OidcTransportError(false);
|
||||
declaredResponseLength(response);
|
||||
if (!reader) {
|
||||
completed = true;
|
||||
return safeBufferedResponse(response, Buffer.alloc(0));
|
||||
}
|
||||
while (true) {
|
||||
const { done, value } = await awaitWithAbort(reader.read(), signal);
|
||||
if (done) break;
|
||||
if (value.byteLength > MAX_OIDC_RESPONSE_BYTES - total) throw new OidcTransportError(false);
|
||||
total += value.byteLength;
|
||||
chunks.push(Buffer.from(value));
|
||||
}
|
||||
completed = true;
|
||||
return safeBufferedResponse(response, Buffer.concat(chunks, total));
|
||||
} finally {
|
||||
try {
|
||||
if (!completed && reader) cancelReaderBestEffort(reader);
|
||||
} finally {
|
||||
try { reader?.releaseLock(); } catch { /* cancellation already made the response unusable */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createBoundedOidcTransport(
|
||||
fetchImplementation: typeof globalThis.fetch,
|
||||
defaultTimeoutMs: number,
|
||||
): BoundedOidcTransport {
|
||||
const request: BoundedOidcTransport["request"] = async (input, init, requestOptions) => {
|
||||
let target: URL;
|
||||
try {
|
||||
target = providerRequestUrl(input);
|
||||
} catch {
|
||||
throw new OidcTransportError(false);
|
||||
}
|
||||
const timeoutMs = requestOptions?.timeoutMs ?? defaultTimeoutMs;
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
timeout.unref();
|
||||
const signal = init?.signal ? AbortSignal.any([init.signal, controller.signal]) : controller.signal;
|
||||
try {
|
||||
const requestInit: RequestInit = { ...init, redirect: "manual", signal };
|
||||
const response = await awaitWithAbort(
|
||||
Promise.resolve().then(() => fetchImplementation(target, requestInit)),
|
||||
signal,
|
||||
cancelResponseBestEffort,
|
||||
);
|
||||
return await boundedResponse(response, signal, requestOptions?.requireSuccess === true);
|
||||
} catch (error) {
|
||||
if (error instanceof OidcTransportError) throw error;
|
||||
if (signal.aborted) throw new OidcTransportError(true);
|
||||
throw new OidcTransportError(true);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
};
|
||||
return {
|
||||
request,
|
||||
// openid-client's FetchBody is Fetch-compatible, but comes from a distinct declaration graph.
|
||||
customFetch: (url, options) => request(url, options as unknown as RequestInit),
|
||||
};
|
||||
}
|
||||
|
||||
function availabilityFailure(error: unknown): boolean {
|
||||
let current = error;
|
||||
const seen = new Set<object>();
|
||||
for (let depth = 0; depth < 8; depth += 1) {
|
||||
if (current instanceof OidcTransportError) return current.availability;
|
||||
if (!current || typeof current !== "object" || seen.has(current)) return false;
|
||||
seen.add(current);
|
||||
current = (current as { cause?: unknown }).cause;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function protocolFailure(error: unknown): OidcProtocolError {
|
||||
let current = error;
|
||||
const seen = new Set<object>();
|
||||
for (let depth = 0; depth < 8; depth += 1) {
|
||||
if (current instanceof OidcProtocolError) return current;
|
||||
if (!current || typeof current !== "object" || seen.has(current)) break;
|
||||
seen.add(current);
|
||||
current = (current as { cause?: unknown }).cause;
|
||||
}
|
||||
return availabilityFailure(error) ? new OidcProviderUnavailableError() : new OidcProtocolError();
|
||||
}
|
||||
|
||||
async function verifyIdTokenSignature(
|
||||
idToken: unknown,
|
||||
config: Configuration,
|
||||
transport: BoundedOidcTransport,
|
||||
jwksTimeoutMs: number,
|
||||
): Promise<void> {
|
||||
if (!text(idToken, MAX_ID_TOKEN_LENGTH)) throw new OidcProtocolError();
|
||||
const [protectedPart, payloadPart, signaturePart, extra] = idToken.split(".");
|
||||
if (!protectedPart || !payloadPart || !signaturePart || extra) throw new OidcProtocolError();
|
||||
const header = jsonPart(protectedPart);
|
||||
if (!text(header.alg, 16) || !text(header.kid, 256)) throw new OidcProtocolError();
|
||||
const metadata = config.serverMetadata();
|
||||
if (!Array.isArray(metadata.id_token_signing_alg_values_supported)
|
||||
|| !metadata.id_token_signing_alg_values_supported.includes(header.alg)
|
||||
|| !text(metadata.jwks_uri, 2048)) throw new OidcProtocolError();
|
||||
const jwksUrl = httpsEndpoint(metadata.jwks_uri);
|
||||
try {
|
||||
const response = await transport.request(
|
||||
jwksUrl,
|
||||
{ headers: { accept: "application/json" }, redirect: "manual" },
|
||||
{ timeoutMs: jwksTimeoutMs, requireSuccess: true },
|
||||
);
|
||||
const body = new TextDecoder("utf-8", { fatal: true }).decode(await response.arrayBuffer());
|
||||
const parsed = JSON.parse(body) as { keys?: unknown };
|
||||
if (!Array.isArray(parsed.keys) || parsed.keys.length === 0 || parsed.keys.length > 16) throw new OidcProtocolError();
|
||||
const matching = parsed.keys.filter((key): key is Record<string, unknown> =>
|
||||
Boolean(key) && typeof key === "object" && !Array.isArray(key) && key.kid === header.kid);
|
||||
if (matching.length !== 1) throw new OidcProtocolError();
|
||||
const key = matching[0];
|
||||
if (key.use !== undefined && key.use !== "sig") throw new OidcProtocolError();
|
||||
if (key.alg !== undefined && key.alg !== header.alg) throw new OidcProtocolError();
|
||||
const algorithm = signatureAlgorithm(header.alg);
|
||||
if (!/^[A-Za-z0-9_-]+$/.test(signaturePart)) throw new OidcProtocolError();
|
||||
const signature = Buffer.from(signaturePart, "base64url");
|
||||
if (signature.length === 0) throw new OidcProtocolError();
|
||||
const publicKey = createPublicKey({ key: key as never, format: "jwk" });
|
||||
const normalizedSignature = algorithm.ecdsaPartLength
|
||||
? joseEcdsaSignatureToDer(signature, algorithm.ecdsaPartLength)
|
||||
: signature;
|
||||
const verified = algorithm.pss
|
||||
? verifySignature(algorithm.digest, Buffer.from(`${protectedPart}.${payloadPart}`), {
|
||||
key: publicKey, padding: constants.RSA_PKCS1_PSS_PADDING, saltLength: algorithm.saltLength,
|
||||
}, normalizedSignature)
|
||||
: verifySignature(algorithm.digest, Buffer.from(`${protectedPart}.${payloadPart}`), publicKey, normalizedSignature);
|
||||
if (!verified) throw new OidcProtocolError();
|
||||
} catch (error) {
|
||||
throw protocolFailure(error);
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyJwksAvailability(
|
||||
config: Configuration,
|
||||
transport: BoundedOidcTransport,
|
||||
jwksTimeoutMs: number,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
const metadata = config.serverMetadata();
|
||||
if (!text(metadata.jwks_uri, 2048)) throw new OidcProtocolError();
|
||||
const response = await transport.request(
|
||||
httpsEndpoint(metadata.jwks_uri),
|
||||
{ headers: { accept: "application/json" }, redirect: "manual", signal },
|
||||
{ timeoutMs: jwksTimeoutMs, requireSuccess: true },
|
||||
);
|
||||
const parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(await response.arrayBuffer()));
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)
|
||||
|| !Array.isArray((parsed as { keys?: unknown }).keys)) throw new OidcProtocolError();
|
||||
}
|
||||
|
||||
export function createOidcProtocol(options: OidcProtocolOptions): OidcProtocol {
|
||||
const issuerUrl = configuredHttpsUrl(options.issuer);
|
||||
const callbackUrl = configuredCallbackUrl(options.callbackUrl);
|
||||
if (!text(options.clientId, 512) || !isUsableAuthenticationSecret("THT_OIDC_CLIENT_SECRET", options.clientSecret)
|
||||
|| !text(options.groupsClaim, 128) || options.scopes.length === 0 || options.scopes.length > 16
|
||||
|| options.scopes.some((scope) => !text(scope, 128))
|
||||
|| (options.httpTimeoutMs !== undefined && (!Number.isSafeInteger(options.httpTimeoutMs)
|
||||
|| options.httpTimeoutMs < 1 || options.httpTimeoutMs > MAX_HTTP_TIMEOUT_MS))
|
||||
|| (options.jwksTimeoutMs !== undefined && (!Number.isSafeInteger(options.jwksTimeoutMs)
|
||||
|| options.jwksTimeoutMs < 1 || options.jwksTimeoutMs > MAX_JWKS_TIMEOUT_MS))) throw new OidcProtocolError();
|
||||
const httpTimeoutMs = options.httpTimeoutMs ?? DEFAULT_HTTP_TIMEOUT_MS;
|
||||
const jwksTimeoutMs = options.jwksTimeoutMs ?? DEFAULT_JWKS_TIMEOUT_MS;
|
||||
const transport = createBoundedOidcTransport(options.fetch ?? globalThis.fetch, httpTimeoutMs);
|
||||
|
||||
let discovered: Promise<Configuration> | undefined;
|
||||
const configuration = async (): Promise<Configuration> => {
|
||||
if (!discovered) {
|
||||
discovered = (async () => {
|
||||
let certifiedIssuerMismatch = false;
|
||||
const issuerCheckingFetch: CustomFetch = async (input, init) => {
|
||||
const response = await transport.customFetch(input, init);
|
||||
if (!response.ok) return response;
|
||||
try {
|
||||
const metadata: unknown = await response.clone().json();
|
||||
if (schemaValidDiscoveryMetadata(metadata) && metadata.issuer !== options.issuer) {
|
||||
certifiedIssuerMismatch = true;
|
||||
const headers = new Headers(response.headers);
|
||||
headers.delete("content-length");
|
||||
return new Response(JSON.stringify({ ...metadata, issuer: options.issuer }), {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// The OIDC library owns malformed discovery-document classification.
|
||||
}
|
||||
return response;
|
||||
};
|
||||
try {
|
||||
const config = await discovery(
|
||||
issuerUrl,
|
||||
options.clientId,
|
||||
{ client_secret: options.clientSecret, redirect_uris: [callbackUrl.href], response_types: ["code"] },
|
||||
undefined,
|
||||
{ [customFetch]: issuerCheckingFetch, timeout: httpTimeoutMs / 1000 },
|
||||
);
|
||||
const metadata = config.serverMetadata();
|
||||
if (metadata.issuer !== options.issuer) throw new OidcProtocolError();
|
||||
httpsEndpoint(metadata.authorization_endpoint);
|
||||
httpsEndpoint(metadata.token_endpoint);
|
||||
httpsEndpoint(metadata.jwks_uri);
|
||||
if (certifiedIssuerMismatch) throw new OidcIssuerMismatchError();
|
||||
return config;
|
||||
} catch (error) {
|
||||
throw protocolFailure(error);
|
||||
}
|
||||
})();
|
||||
}
|
||||
return await discovered;
|
||||
};
|
||||
|
||||
return {
|
||||
async authorizationUrl(input) {
|
||||
try {
|
||||
const challenge = await calculatePKCECodeChallenge(input.codeVerifier);
|
||||
return buildAuthorizationUrl(await configuration(), {
|
||||
response_type: "code",
|
||||
redirect_uri: callbackUrl.href,
|
||||
scope: options.scopes.join(" "),
|
||||
state: input.state,
|
||||
nonce: input.nonce,
|
||||
code_challenge: challenge,
|
||||
code_challenge_method: "S256",
|
||||
});
|
||||
} catch (error) {
|
||||
throw protocolFailure(error);
|
||||
}
|
||||
},
|
||||
async callback(input) {
|
||||
if (input.currentUrl.origin !== callbackUrl.origin || input.currentUrl.pathname !== callbackUrl.pathname) {
|
||||
throw new OidcProtocolError();
|
||||
}
|
||||
try {
|
||||
const config = await configuration();
|
||||
const tokens = await authorizationCodeGrant(config, input.currentUrl, {
|
||||
expectedState: input.state,
|
||||
expectedNonce: input.nonce,
|
||||
pkceCodeVerifier: input.codeVerifier,
|
||||
idTokenExpected: true,
|
||||
});
|
||||
await verifyIdTokenSignature(tokens.id_token, config, transport, jwksTimeoutMs);
|
||||
const claims = tokens.claims();
|
||||
if (!claims || Array.isArray(claims)) throw new OidcProtocolError();
|
||||
return identityFromClaims(claims as Record<string, unknown>, options);
|
||||
} catch (error) {
|
||||
throw protocolFailure(error);
|
||||
}
|
||||
},
|
||||
async diagnose(signal) {
|
||||
signal.throwIfAborted();
|
||||
const config = await configuration();
|
||||
signal.throwIfAborted();
|
||||
try {
|
||||
await verifyJwksAvailability(config, transport, jwksTimeoutMs, signal);
|
||||
} catch {
|
||||
throw new OidcJwksUnavailableError();
|
||||
}
|
||||
signal.throwIfAborted();
|
||||
},
|
||||
async verifyDeviceFlow(signal, present) {
|
||||
const maximumDeadline = AbortSignal.timeout(MAX_DEVICE_FLOW_TIMEOUT_MS);
|
||||
const operationSignal = AbortSignal.any([signal, maximumDeadline]);
|
||||
let config: Configuration;
|
||||
try {
|
||||
operationSignal.throwIfAborted();
|
||||
config = await configuration();
|
||||
operationSignal.throwIfAborted();
|
||||
const endpoint = config.serverMetadata().device_authorization_endpoint;
|
||||
httpsEndpoint(endpoint);
|
||||
} catch (error) {
|
||||
if (error instanceof OidcIssuerMismatchError || error instanceof OidcProviderUnavailableError) throw error;
|
||||
throw new OidcDeviceFlowUnavailableError();
|
||||
}
|
||||
try {
|
||||
operationSignal.throwIfAborted();
|
||||
const device = await awaitWithAbort(
|
||||
initiateDeviceAuthorization(config, { scope: options.scopes.join(" ") }),
|
||||
operationSignal,
|
||||
);
|
||||
if (!text(device.verification_uri, 2048) || !text(device.user_code, 256)) {
|
||||
throw new OidcDeviceFlowUnavailableError();
|
||||
}
|
||||
const providerLifetimeMs = device.expires_in * 1000;
|
||||
if (!Number.isSafeInteger(providerLifetimeMs) || providerLifetimeMs <= 0) {
|
||||
throw new OidcDeviceFlowUnavailableError();
|
||||
}
|
||||
const deviceSignal = AbortSignal.any([
|
||||
signal,
|
||||
maximumDeadline,
|
||||
AbortSignal.timeout(Math.min(providerLifetimeMs, MAX_DEVICE_FLOW_TIMEOUT_MS)),
|
||||
]);
|
||||
const verificationUri = httpsEndpoint(device.verification_uri);
|
||||
present(verificationUri.href, device.user_code);
|
||||
const tokens = await pollDeviceAuthorizationGrant(config, device, undefined, { signal: deviceSignal });
|
||||
await verifyIdTokenSignature(tokens.id_token, config, transport, jwksTimeoutMs);
|
||||
const claims = tokens.claims();
|
||||
if (!claims || Array.isArray(claims)) throw new OidcProtocolError();
|
||||
return identityFromClaims(claims as Record<string, unknown>, options);
|
||||
} catch (error) {
|
||||
if (error instanceof OidcDeviceFlowUnavailableError) throw error;
|
||||
throw protocolFailure(error);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
import { argon2, timingSafeEqual } from "node:crypto";
|
||||
|
||||
const MAXIMUM_PHC_BYTES = 256;
|
||||
const ARGON2_MEMORY_KIB = 65_536;
|
||||
const ARGON2_PASSES = 3;
|
||||
const ARGON2_PARALLELISM = 1;
|
||||
const ARGON2_SALT_BYTES = 16;
|
||||
const ARGON2_KEY_BYTES = 32;
|
||||
const MINIMUM_PASSWORD_BYTES = 12;
|
||||
const MAXIMUM_PASSWORD_BYTES = 1024;
|
||||
|
||||
interface Argon2Parameters {
|
||||
memory: number;
|
||||
passes: number;
|
||||
parallelism: number;
|
||||
salt: Buffer;
|
||||
digest: Buffer;
|
||||
}
|
||||
|
||||
/** Internal-only signal for unavailable native Argon2 work; never expose its cause. */
|
||||
export class LocalPasswordVerificationError extends Error {
|
||||
constructor() {
|
||||
super("local_password_verification_failed");
|
||||
}
|
||||
}
|
||||
|
||||
function parseDecimal(value: string, maximum: number): number | undefined {
|
||||
if (!/^\d+$/.test(value) || (value.length > 1 && value[0] === "0") || value.length > 10) return undefined;
|
||||
const parsed = Number(value);
|
||||
return Number.isSafeInteger(parsed) && parsed <= maximum ? parsed : undefined;
|
||||
}
|
||||
|
||||
function decodeRawBase64(value: string, minimum: number, maximum: number): Buffer | undefined {
|
||||
if (!/^[A-Za-z0-9+/]+$/.test(value)) return undefined;
|
||||
const decoded = Buffer.from(value, "base64");
|
||||
if (decoded.length < minimum || decoded.length > maximum) {
|
||||
decoded.fill(0);
|
||||
return undefined;
|
||||
}
|
||||
if (decoded.toString("base64").replace(/=+$/, "") !== value) {
|
||||
decoded.fill(0);
|
||||
return undefined;
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
function parsePHC(encoded: string): Argon2Parameters | undefined {
|
||||
if (typeof encoded !== "string" || encoded.length === 0 || Buffer.byteLength(encoded, "utf8") > MAXIMUM_PHC_BYTES) return undefined;
|
||||
const parts = encoded.split("$");
|
||||
if (parts.length !== 6 || parts[0] !== "" || parts[1] !== "argon2id" || parts[2] !== "v=19") return undefined;
|
||||
|
||||
const parameterParts = parts[3].split(",");
|
||||
if (parameterParts.length !== 3 || !parameterParts[0].startsWith("m=") || !parameterParts[1].startsWith("t=") || !parameterParts[2].startsWith("p=")) return undefined;
|
||||
const memory = parseDecimal(parameterParts[0].slice(2), ARGON2_MEMORY_KIB);
|
||||
const passes = parseDecimal(parameterParts[1].slice(2), ARGON2_PASSES);
|
||||
const parallelism = parseDecimal(parameterParts[2].slice(2), ARGON2_PARALLELISM);
|
||||
if (memory !== ARGON2_MEMORY_KIB || passes !== ARGON2_PASSES || parallelism !== ARGON2_PARALLELISM) return undefined;
|
||||
|
||||
const salt = decodeRawBase64(parts[4], ARGON2_SALT_BYTES, ARGON2_SALT_BYTES);
|
||||
const digest = decodeRawBase64(parts[5], ARGON2_KEY_BYTES, ARGON2_KEY_BYTES);
|
||||
if (!salt || !digest) {
|
||||
salt?.fill(0);
|
||||
digest?.fill(0);
|
||||
return undefined;
|
||||
}
|
||||
return { memory, passes, parallelism, salt, digest };
|
||||
}
|
||||
|
||||
function hasValidPasswordBytes(password: string): boolean {
|
||||
if (typeof password !== "string") return false;
|
||||
const typedPassword = password as string & { isWellFormed?: () => boolean };
|
||||
if (typeof typedPassword.isWellFormed === "function") {
|
||||
if (!typedPassword.isWellFormed()) return false;
|
||||
} else if (/[\uD800-\uDFFF]/.test(password)) {
|
||||
return false;
|
||||
}
|
||||
const byteLength = Buffer.byteLength(password, "utf8");
|
||||
return byteLength >= MINIMUM_PASSWORD_BYTES && byteLength <= MAXIMUM_PASSWORD_BYTES;
|
||||
}
|
||||
|
||||
function passwordBytes(password: string): Buffer | undefined {
|
||||
return hasValidPasswordBytes(password) ? Buffer.from(password, "utf8") : undefined;
|
||||
}
|
||||
|
||||
function clearParameters(parameters: Argon2Parameters): void {
|
||||
parameters.salt.fill(0);
|
||||
parameters.digest.fill(0);
|
||||
}
|
||||
|
||||
function deriveArgon2(message: Buffer, parameters: Argon2Parameters): Promise<Buffer> {
|
||||
return new Promise<Buffer>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const complete = (error: Error | null, derived?: Buffer): void => {
|
||||
if (settled) {
|
||||
derived?.fill(0);
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
if (error || !derived) {
|
||||
derived?.fill(0);
|
||||
reject(error ?? new Error("argon2_failed"));
|
||||
return;
|
||||
}
|
||||
resolve(derived);
|
||||
};
|
||||
try {
|
||||
argon2("argon2id", {
|
||||
message,
|
||||
nonce: parameters.salt,
|
||||
memory: parameters.memory,
|
||||
passes: parameters.passes,
|
||||
parallelism: parameters.parallelism,
|
||||
tagLength: parameters.digest.length,
|
||||
}, complete);
|
||||
} catch (error) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
reject(error);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function isValidPasswordHash(encoded: string): boolean {
|
||||
const parameters = parsePHC(encoded);
|
||||
if (!parameters) return false;
|
||||
clearParameters(parameters);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function verifyPassword(password: string, encoded: string): Promise<boolean> {
|
||||
const parameters = parsePHC(encoded);
|
||||
const message = passwordBytes(password);
|
||||
if (!message || !parameters) {
|
||||
message?.fill(0);
|
||||
if (parameters) clearParameters(parameters);
|
||||
return false;
|
||||
}
|
||||
let derived: Buffer | undefined;
|
||||
try {
|
||||
derived = await deriveArgon2(message, parameters);
|
||||
return derived.length === parameters.digest.length && timingSafeEqual(derived, parameters.digest);
|
||||
} catch {
|
||||
throw new LocalPasswordVerificationError();
|
||||
} finally {
|
||||
message.fill(0);
|
||||
derived?.fill(0);
|
||||
clearParameters(parameters);
|
||||
}
|
||||
}
|
||||
|
||||
const DUMMY_PASSWORD = "thothii-process-local-dummy-password";
|
||||
const DUMMY_HASH = "$argon2id$v=19$m=65536,t=3,p=1$ABEiM0RVZneImaq7zN3u/w$/YuK14HV5biXcVIYqaJs07meu0nRgo+d62KnM02MSoU";
|
||||
|
||||
export async function verifyWithDummy(password: string): Promise<void> {
|
||||
await verifyPassword(hasValidPasswordBytes(password) ? password : DUMMY_PASSWORD, DUMMY_HASH);
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { isPermission, rolesToPermissions } from "./config.js";
|
||||
import type { Permission, Role } from "./types.js";
|
||||
|
||||
export interface PrincipalContext {
|
||||
issuer: string;
|
||||
subject: string;
|
||||
displayName?: string;
|
||||
roles: readonly Role[];
|
||||
permissions: readonly Permission[];
|
||||
isAdmin: boolean;
|
||||
}
|
||||
|
||||
const principalEnvKeys = [
|
||||
"THT_PRINCIPAL_ISSUER", "THT_PRINCIPAL_SUBJECT", "THT_PRINCIPAL_DISPLAY_NAME", "THT_PRINCIPAL_IS_ADMIN",
|
||||
"THT_PRINCIPAL_PERMISSIONS",
|
||||
] as const;
|
||||
|
||||
export function clearPrincipalEnvironment(env: NodeJS.ProcessEnv): void {
|
||||
for (const key of principalEnvKeys) delete env[key];
|
||||
}
|
||||
|
||||
export function expandLocalHome(path: string, home = homedir()): string {
|
||||
if (path === "~") return home;
|
||||
if (path.startsWith("~/")) return join(home, path.slice(2));
|
||||
return path;
|
||||
}
|
||||
|
||||
function harden(path: string, mode: number): void {
|
||||
if (process.platform === "win32") return;
|
||||
try { chmodSync(path, mode); } catch { /* best-effort parity with harness local storage */ }
|
||||
}
|
||||
|
||||
const invalid = (value: string) => value.length === 0 || value.length > 512 || /[\u0000-\u001f\u007f]/.test(value);
|
||||
|
||||
function required(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const normalized = value.trim();
|
||||
return invalid(normalized) ? undefined : normalized;
|
||||
}
|
||||
|
||||
function optional(value: unknown): string | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
return required(value);
|
||||
}
|
||||
|
||||
function principal(
|
||||
issuer: string, subject: string, roles: readonly Role[], displayName?: string,
|
||||
): PrincipalContext {
|
||||
return {
|
||||
issuer,
|
||||
subject,
|
||||
...(displayName ? { displayName } : {}),
|
||||
roles,
|
||||
permissions: rolesToPermissions(roles),
|
||||
isAdmin: roles.includes("admin"),
|
||||
};
|
||||
}
|
||||
|
||||
export function upstreamPrincipal(headers: Record<string, unknown>): PrincipalContext | undefined {
|
||||
const issuer = required(headers["x-thoth-principal-issuer"]);
|
||||
const subject = required(headers["x-thoth-principal-subject"]);
|
||||
const displayName = optional(headers["x-thoth-principal-display-name"]);
|
||||
const adminHeader = headers["x-thoth-is-admin"];
|
||||
if (!issuer || !subject || (headers["x-thoth-principal-display-name"] !== undefined && !displayName)) return undefined;
|
||||
if (adminHeader !== "0" && adminHeader !== "1" && adminHeader !== "true" && adminHeader !== "false") return undefined;
|
||||
return principal(
|
||||
issuer,
|
||||
subject,
|
||||
adminHeader === "1" || adminHeader === "true" ? ["user", "admin"] : ["user"],
|
||||
displayName,
|
||||
);
|
||||
}
|
||||
|
||||
export function localPrincipal(publicExposure = false): PrincipalContext {
|
||||
const home = expandLocalHome(process.env.THT_HOME ?? join(homedir(), ".thothii"));
|
||||
const identityPath = join(home, "identity.json");
|
||||
mkdirSync(home, { recursive: true, mode: 0o700 });
|
||||
harden(home, 0o700);
|
||||
try {
|
||||
const stored = JSON.parse(readFileSync(identityPath, "utf8"));
|
||||
if (stored?.issuer === "local" && typeof stored.subject === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(stored.subject)) {
|
||||
harden(identityPath, 0o600);
|
||||
return principal("local", stored.subject, publicExposure ? ["user"] : ["admin"]);
|
||||
}
|
||||
throw new Error("invalid local identity");
|
||||
} catch (error: any) {
|
||||
if (error?.code !== "ENOENT") throw error;
|
||||
const created = { issuer: "local", subject: randomUUID() };
|
||||
try {
|
||||
writeFileSync(identityPath, JSON.stringify(created) + "\n", { mode: 0o600, flag: "wx" });
|
||||
harden(identityPath, 0o600);
|
||||
return principalContext("local", created.subject, publicExposure);
|
||||
} catch (writeError: any) {
|
||||
// Another local request won the identity creation race; always converge on its UUID.
|
||||
if (writeError?.code === "EEXIST") return localPrincipal(publicExposure);
|
||||
throw writeError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function principalContext(issuer: string, subject: string, publicExposure: boolean): PrincipalContext {
|
||||
return principal(issuer, subject, publicExposure ? ["user"] : ["admin"]);
|
||||
}
|
||||
|
||||
export function principalEnvironment(principal: PrincipalContext): NodeJS.ProcessEnv {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
THT_PRINCIPAL_ISSUER: principal.issuer,
|
||||
THT_PRINCIPAL_SUBJECT: principal.subject,
|
||||
THT_PRINCIPAL_IS_ADMIN: principal.isAdmin ? "true" : "false",
|
||||
THT_PRINCIPAL_PERMISSIONS: principal.permissions.filter(isPermission).join(","),
|
||||
};
|
||||
if (principal.displayName) env.THT_PRINCIPAL_DISPLAY_NAME = principal.displayName;
|
||||
return env;
|
||||
}
|
||||
@@ -1,665 +0,0 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
||||
import type {
|
||||
AuthenticationConfigProvider,
|
||||
LoadedAuthConfig,
|
||||
LocalUserRecord,
|
||||
OidcAuthenticationConfig,
|
||||
OidcStateRecord,
|
||||
OidcTransactionTransport,
|
||||
Role,
|
||||
} from "./types.js";
|
||||
import type { LocalUserRegistry } from "./local-registry.js";
|
||||
import { OidcStateCapacityError, type AuthSessionStore } from "./session-store.js";
|
||||
import { rolesToPermissions } from "./config.js";
|
||||
import { captureAuthConfigSnapshot, getPrincipal, requireExactOrigin, sessionCookieName } from "./auth.js";
|
||||
import { requirePermission, isPrincipalContext } from "./authorization.js";
|
||||
import { deriveCsrfToken } from "./csrf.js";
|
||||
import { verifyWithDummy } from "./password.js";
|
||||
import type { OidcProtocol } from "./oidc-client.js";
|
||||
import { parseConfiguredTransportUrl } from "./url-policy.js";
|
||||
|
||||
const TEN_MINUTES_MS = 10 * 60 * 1000;
|
||||
const REMEMBER_COOKIE_SECONDS = 2_592_000;
|
||||
const MAX_USERNAME_LENGTH = 64;
|
||||
const MAX_PASSWORD_LENGTH = 1024;
|
||||
const MAX_LIMIT_ENTRIES = 10_000;
|
||||
const MAX_OIDC_INITIATIONS_PER_ADDRESS = 20;
|
||||
const MAX_OIDC_CALLBACK_QUERY_LENGTH = 4096;
|
||||
const OIDC_CALLBACK_PATH = "/api/auth/oidc/callback";
|
||||
const OIDC_TRANSACTION_COOKIE_SECONDS = TEN_MINUTES_MS / 1000;
|
||||
const OIDC_VALUE_PATTERN = /^[A-Za-z0-9_-]{43}$/;
|
||||
|
||||
interface OidcTransactionCookieProfile {
|
||||
transport: OidcTransactionTransport;
|
||||
name: string;
|
||||
secure: boolean;
|
||||
}
|
||||
|
||||
const OIDC_TRANSACTION_COOKIE_PROFILES: Readonly<Record<OidcTransactionTransport, OidcTransactionCookieProfile>> = {
|
||||
https: { transport: "https", name: "__Host-thothii_oidc_tx", secure: true },
|
||||
loopback_http: { transport: "loopback_http", name: "thothii_oidc_tx", secure: false },
|
||||
};
|
||||
|
||||
export interface AuthRouteDependencies {
|
||||
authMode: "local" | "oidc" | "upstream" | "none" | "mock";
|
||||
authentication?: AuthenticationConfigProvider;
|
||||
sessionStore?: AuthSessionStore;
|
||||
/** Test-only compatibility seam; production resolves from each loaded config snapshot. */
|
||||
localUserRegistry?: LocalUserRegistry;
|
||||
resolveLocalUserRegistry?: (loaded: LoadedAuthConfig) => LocalUserRegistry | undefined;
|
||||
oidcProtocol?: OidcProtocol;
|
||||
resolveOidcProtocol?: (loaded: LoadedAuthConfig) => OidcProtocol | undefined;
|
||||
}
|
||||
|
||||
interface LoginPayload {
|
||||
username: string;
|
||||
password: string;
|
||||
remember: boolean;
|
||||
}
|
||||
|
||||
function countActiveAttempts(
|
||||
bucket: ReadonlyMap<string, readonly number[]>,
|
||||
key: string,
|
||||
now: number,
|
||||
): number {
|
||||
const attempts = bucket.get(key);
|
||||
if (!attempts) return 0;
|
||||
const earliest = now - TEN_MINUTES_MS;
|
||||
let count = 0;
|
||||
for (const timestamp of attempts) {
|
||||
if (timestamp > earliest) count += 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function pruneExpiredAttempts(bucket: Map<string, number[]>, now: number): void {
|
||||
const earliest = now - TEN_MINUTES_MS;
|
||||
for (const [key, attempts] of bucket) {
|
||||
const active = attempts.filter((timestamp) => timestamp > earliest);
|
||||
if (active.length === 0) bucket.delete(key);
|
||||
else if (active.length !== attempts.length) bucket.set(key, active);
|
||||
}
|
||||
}
|
||||
|
||||
function canRecordAttempt(
|
||||
bucket: ReadonlyMap<string, readonly number[]>,
|
||||
key: string,
|
||||
limit: number,
|
||||
maximumEntries: number,
|
||||
): boolean {
|
||||
return (bucket.get(key)?.length ?? 0) < limit
|
||||
&& (bucket.has(key) || bucket.size < maximumEntries);
|
||||
}
|
||||
|
||||
function appendAttempt(bucket: Map<string, number[]>, key: string, now: number): void {
|
||||
bucket.set(key, [...(bucket.get(key) ?? []), now]);
|
||||
}
|
||||
|
||||
export class LoginFailureLimiter {
|
||||
private readonly usernames = new Map<string, number[]>();
|
||||
private readonly addresses = new Map<string, number[]>();
|
||||
private readonly maximumEntries: number;
|
||||
|
||||
constructor(options: { maximumEntries?: number } = {}) {
|
||||
this.maximumEntries = options.maximumEntries ?? MAX_LIMIT_ENTRIES;
|
||||
}
|
||||
|
||||
isLimited(username: string, address: string, now = Date.now()): boolean {
|
||||
return countActiveAttempts(this.usernames, username, now) >= 10
|
||||
|| countActiveAttempts(this.addresses, address, now) >= 20;
|
||||
}
|
||||
|
||||
recordFailure(username: string, address: string, now = Date.now()): boolean {
|
||||
pruneExpiredAttempts(this.usernames, now);
|
||||
pruneExpiredAttempts(this.addresses, now);
|
||||
if (!canRecordAttempt(this.usernames, username, 10, this.maximumEntries)
|
||||
|| !canRecordAttempt(this.addresses, address, 20, this.maximumEntries)) return false;
|
||||
appendAttempt(this.usernames, username, now);
|
||||
appendAttempt(this.addresses, address, now);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
class OidcInitiationLimiter {
|
||||
private readonly addresses = new Map<string, number[]>();
|
||||
|
||||
consume(address: string, now = Date.now()): boolean {
|
||||
pruneExpiredAttempts(this.addresses, now);
|
||||
if (!canRecordAttempt(
|
||||
this.addresses,
|
||||
address,
|
||||
MAX_OIDC_INITIATIONS_PER_ADDRESS,
|
||||
MAX_LIMIT_ENTRIES,
|
||||
)) return false;
|
||||
appendAttempt(this.addresses, address, now);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
class VerificationGate {
|
||||
private active = 0;
|
||||
|
||||
async run(operation: () => Promise<boolean>): Promise<boolean | undefined> {
|
||||
if (this.active >= 2) return undefined;
|
||||
this.active += 1;
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
this.active -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function unavailableAfterDummy(
|
||||
gate: VerificationGate,
|
||||
password: string,
|
||||
reply: FastifyReply,
|
||||
): Promise<FastifyReply> {
|
||||
try {
|
||||
const completed = await gate.run(async () => {
|
||||
await verifyWithDummy(password);
|
||||
return true;
|
||||
});
|
||||
if (completed === undefined) return loginLimited(reply);
|
||||
} catch {
|
||||
// Preserve the sanitized operational outcome below.
|
||||
}
|
||||
return unavailable(reply);
|
||||
}
|
||||
|
||||
export function registerAuthRoutes(app: FastifyInstance, deps: AuthRouteDependencies): void {
|
||||
const limiter = new LoginFailureLimiter();
|
||||
const oidcInitiationLimiter = new OidcInitiationLimiter();
|
||||
const verificationGate = new VerificationGate();
|
||||
|
||||
app.get("/auth/config", async (request, reply) => {
|
||||
const snapshot = captureAuthConfigSnapshot(request, deps.authentication);
|
||||
if (!snapshot) {
|
||||
if (deps.authMode === "upstream") {
|
||||
return reply.send({ mode: "upstream", localLogin: false, oidcLogin: false });
|
||||
}
|
||||
return unavailable(reply);
|
||||
}
|
||||
const mode = snapshot.value.mode;
|
||||
return reply.send({ mode, localLogin: mode === "local", oidcLogin: mode === "oidc" });
|
||||
});
|
||||
|
||||
app.post("/auth/local/login", async (request, reply) => {
|
||||
const snapshot = captureAuthConfigSnapshot(request, deps.authentication);
|
||||
const configured = currentLocalConfig(snapshot, deps);
|
||||
if (configured.kind === "unavailable") {
|
||||
return unavailableAfterDummy(verificationGate, argon2SafePassword(loginPayload(request).password), reply);
|
||||
}
|
||||
if (configured.kind !== "local" || !deps.sessionStore) return unavailable(reply);
|
||||
const originCheck = requireExactOrigin(request, reply, configured.origin);
|
||||
if (originCheck !== true) return originCheck;
|
||||
|
||||
const payload = loginPayload(request);
|
||||
const safePassword = argon2SafePassword(payload.password);
|
||||
const normalizedUsername = payload.username.replace(/[A-Z]/g, (character) => character.toLowerCase());
|
||||
const sourceAddress = boundedAddress(request.ip);
|
||||
if (limiter.isLimited(normalizedUsername, sourceAddress)) return loginLimited(reply);
|
||||
|
||||
let user: LocalUserRecord | undefined;
|
||||
try {
|
||||
if (payload.username.length > 0) user = await configured.registry.findByUsername(payload.username);
|
||||
} catch {
|
||||
return unavailableAfterDummy(verificationGate, safePassword, reply);
|
||||
}
|
||||
let verified: boolean | undefined;
|
||||
try {
|
||||
verified = await verificationGate.run(async () =>
|
||||
configured.registry.verify(user, safePassword));
|
||||
} catch {
|
||||
return unavailable(reply);
|
||||
}
|
||||
if (verified === undefined) return loginLimited(reply);
|
||||
if (!verified || !user || !user.enabled) {
|
||||
if (!limiter.recordFailure(normalizedUsername, sourceAddress)) return loginLimited(reply);
|
||||
return invalidCredentials(reply);
|
||||
}
|
||||
|
||||
try {
|
||||
const created = await deps.sessionStore.create({
|
||||
principal: {
|
||||
issuer: "local",
|
||||
subject: user.id,
|
||||
displayName: user.displayName ?? user.username,
|
||||
roles: user.roles,
|
||||
permissions: rolesToPermissions(user.roles),
|
||||
isAdmin: user.roles.includes("admin"),
|
||||
},
|
||||
method: "local",
|
||||
remembered: payload.remember,
|
||||
userAuthRevision: user.authRevision,
|
||||
authConfigRevision: configured.revision,
|
||||
idleTtlMs: (payload.remember ? configured.session.rememberIdleSeconds : configured.session.regularIdleSeconds) * 1000,
|
||||
absoluteTtlMs: (payload.remember ? configured.session.rememberTtlSeconds : configured.session.regularTtlSeconds) * 1000,
|
||||
});
|
||||
reply.setCookie(sessionCookieName(), created.token, cookieOptions(snapshot, payload.remember));
|
||||
return reply.send({});
|
||||
} catch {
|
||||
return unavailable(reply);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/auth/oidc/login", async (request, reply) => {
|
||||
if (!oidcInitiationLimiter.consume(boundedAddress(request.ip))) return loginLimited(reply);
|
||||
const loaded = captureAuthConfigSnapshot(request, deps.authentication);
|
||||
const configured = currentOidcConfig(loaded, deps);
|
||||
const transactionProfile = configured === undefined
|
||||
? undefined
|
||||
: oidcTransactionCookieProfile(configured.loaded);
|
||||
if (!configured || !transactionProfile || !deps.sessionStore) {
|
||||
clearOidcTransactionCookies(reply);
|
||||
return unavailable(reply);
|
||||
}
|
||||
const nonce = randomOidcValue();
|
||||
const codeVerifier = randomOidcValue();
|
||||
const browserTransaction = randomOidcValue();
|
||||
try {
|
||||
const created = await deps.sessionStore.createOidcState({
|
||||
nonce,
|
||||
codeVerifier,
|
||||
returnTo: "/",
|
||||
authConfigRevision: configured.loaded.revision,
|
||||
issuer: configured.config.oidc.issuer,
|
||||
browserTransactionDigest: oidcTransactionDigest(browserTransaction).toString("hex"),
|
||||
browserTransactionTransport: transactionProfile.transport,
|
||||
});
|
||||
try {
|
||||
const location = await configured.protocol.authorizationUrl({ state: created.state, nonce, codeVerifier });
|
||||
clearOidcTransactionCookie(reply, otherOidcTransactionCookieProfile(transactionProfile));
|
||||
reply.setCookie(transactionProfile.name, browserTransaction, oidcTransactionCookieOptions(transactionProfile));
|
||||
return reply.redirect(location.href);
|
||||
} catch {
|
||||
await deps.sessionStore.consumeOidcState(created.state).catch(() => undefined);
|
||||
clearOidcTransactionCookies(reply, transactionProfile);
|
||||
return unavailable(reply);
|
||||
}
|
||||
} catch (error) {
|
||||
clearOidcTransactionCookies(reply, transactionProfile);
|
||||
if (error instanceof OidcStateCapacityError) return loginLimited(reply);
|
||||
return unavailable(reply);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/auth/oidc/callback", async (request, reply) => {
|
||||
const loaded = captureAuthConfigSnapshot(request, deps.authentication);
|
||||
const callback = oidcCallbackUrl(request, loaded?.value.publicUrl);
|
||||
if (!deps.sessionStore || !callback.state) {
|
||||
clearOidcTransactionCookies(reply);
|
||||
return oidcCallbackFailed(reply);
|
||||
}
|
||||
let state: OidcStateRecord | undefined;
|
||||
try {
|
||||
state = await deps.sessionStore.consumeOidcState(callback.state);
|
||||
} catch {
|
||||
clearOidcTransactionCookies(reply);
|
||||
return oidcCallbackFailed(reply);
|
||||
}
|
||||
const stateTransactionProfile = oidcTransactionCookieProfileForTransport(state?.browserTransactionTransport);
|
||||
clearOidcTransactionCookies(reply, stateTransactionProfile);
|
||||
const configured = currentOidcConfig(loaded, deps);
|
||||
const configuredTransactionProfile = configured === undefined
|
||||
? undefined
|
||||
: oidcTransactionCookieProfile(configured.loaded);
|
||||
const transaction = readOidcTransactionCookie(request, stateTransactionProfile);
|
||||
const transactionMatches = oidcTransactionMatches(transaction, state?.browserTransactionDigest ?? "");
|
||||
if (!callback.currentUrl || !configured || !configuredTransactionProfile || !state || !stateTransactionProfile
|
||||
|| state.returnTo !== "/" || !transactionMatches
|
||||
|| state.authConfigRevision !== configured.loaded.revision
|
||||
|| state.issuer !== configured.config.oidc.issuer
|
||||
|| stateTransactionProfile.transport !== configuredTransactionProfile.transport) {
|
||||
return oidcCallbackFailed(reply);
|
||||
}
|
||||
try {
|
||||
const identity = await configured.protocol.callback({
|
||||
currentUrl: callback.currentUrl,
|
||||
state: callback.state,
|
||||
nonce: state.nonce,
|
||||
codeVerifier: state.codeVerifier,
|
||||
});
|
||||
if (identity.issuer !== configured.config.oidc.issuer || !Array.isArray(identity.groups)
|
||||
|| identity.groups.length === 0) return oidcCallbackFailed(reply);
|
||||
const roles = oidcRoles(identity.groups, configured.config);
|
||||
const now = new Date();
|
||||
const absoluteTtlMs = Math.min(
|
||||
configured.config.session.oidcTtlSeconds * 1000,
|
||||
identity.tokenExpiresAt.getTime() - now.getTime(),
|
||||
);
|
||||
if (!Number.isSafeInteger(absoluteTtlMs) || absoluteTtlMs <= 0) return oidcCallbackFailed(reply);
|
||||
const created = await deps.sessionStore.create({
|
||||
principal: {
|
||||
issuer: identity.issuer,
|
||||
subject: identity.subject,
|
||||
...(identity.displayName === undefined ? {} : { displayName: identity.displayName }),
|
||||
roles,
|
||||
permissions: rolesToPermissions(roles),
|
||||
isAdmin: roles.includes("admin"),
|
||||
},
|
||||
method: "oidc",
|
||||
remembered: false,
|
||||
authConfigRevision: configured.loaded.revision,
|
||||
idleTtlMs: configured.config.session.regularIdleSeconds * 1000,
|
||||
absoluteTtlMs,
|
||||
}, now);
|
||||
reply.setCookie(sessionCookieName(), created.token, cookieOptions(configured.loaded, false));
|
||||
return reply.redirect(state.returnTo);
|
||||
} catch {
|
||||
return oidcCallbackFailed(reply);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/auth/logout", async (request, reply) => {
|
||||
const token = request.authSessionToken;
|
||||
if (!token || !deps.sessionStore) return unavailable(reply);
|
||||
try {
|
||||
await deps.sessionStore.revoke(token);
|
||||
reply.clearCookie(sessionCookieName(), cookieOptions(request.authConfigSnapshot, false));
|
||||
return reply.code(204).send();
|
||||
} catch {
|
||||
return unavailable(reply);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/me", async (request, reply) => {
|
||||
const principal = requirePermission(request, reply, "session.use");
|
||||
if (!isPrincipalContext(principal)) return principal;
|
||||
const session = request.authSession;
|
||||
const token = request.authSessionToken;
|
||||
if (!session || !token) {
|
||||
return {
|
||||
issuer: principal.issuer,
|
||||
subject: principal.subject,
|
||||
...(principal.displayName === undefined ? {} : { displayName: principal.displayName }),
|
||||
roles: principal.roles,
|
||||
permissions: principal.permissions,
|
||||
isAdmin: principal.isAdmin,
|
||||
csrfToken: null,
|
||||
session: null,
|
||||
};
|
||||
}
|
||||
try {
|
||||
return {
|
||||
issuer: principal.issuer,
|
||||
subject: principal.subject,
|
||||
...(principal.displayName === undefined ? {} : { displayName: principal.displayName }),
|
||||
roles: principal.roles,
|
||||
permissions: principal.permissions,
|
||||
isAdmin: principal.isAdmin,
|
||||
csrfToken: deriveCsrfToken(token),
|
||||
session: {
|
||||
method: session.method,
|
||||
remembered: session.remembered,
|
||||
idleExpiresAt: session.idleExpiresAt,
|
||||
absoluteExpiresAt: session.absoluteExpiresAt,
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return unavailable(reply);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function currentLocalConfig(loaded: LoadedAuthConfig | undefined, deps: AuthRouteDependencies):
|
||||
| {
|
||||
revision: string;
|
||||
origin: string;
|
||||
session: { regularTtlSeconds: number; regularIdleSeconds: number; rememberTtlSeconds: number; rememberIdleSeconds: number };
|
||||
registry: LocalUserRegistry;
|
||||
kind: "local";
|
||||
}
|
||||
| { kind: "not_local" }
|
||||
| { kind: "unavailable" } {
|
||||
try {
|
||||
if (!loaded) return { kind: "unavailable" };
|
||||
if (loaded.value.mode !== "local") return { kind: "not_local" };
|
||||
const registry = deps.resolveLocalUserRegistry?.(loaded) ?? deps.localUserRegistry;
|
||||
if (!registry) return { kind: "unavailable" };
|
||||
const url = new URL(loaded.value.publicUrl);
|
||||
return {
|
||||
kind: "local",
|
||||
revision: loaded.revision,
|
||||
origin: url.origin,
|
||||
session: loaded.value.session,
|
||||
registry,
|
||||
};
|
||||
} catch {
|
||||
return { kind: "unavailable" };
|
||||
}
|
||||
}
|
||||
|
||||
function cookieOptions(snapshot: LoadedAuthConfig | undefined, remembered: boolean) {
|
||||
let secure = false;
|
||||
try {
|
||||
secure = snapshot !== undefined && new URL(snapshot.value.publicUrl).protocol === "https:";
|
||||
} catch {
|
||||
// Invalid auth configurations are rejected before they can reach this route.
|
||||
}
|
||||
return {
|
||||
httpOnly: true,
|
||||
sameSite: "lax" as const,
|
||||
path: "/",
|
||||
secure,
|
||||
...(remembered ? { maxAge: REMEMBER_COOKIE_SECONDS } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function currentOidcConfig(loaded: LoadedAuthConfig | undefined, deps: AuthRouteDependencies):
|
||||
| { loaded: LoadedAuthConfig; config: OidcAuthenticationConfig; protocol: OidcProtocol }
|
||||
| undefined {
|
||||
if (!loaded || loaded.value.mode !== "oidc") return undefined;
|
||||
const protocol = deps.resolveOidcProtocol?.(loaded) ?? deps.oidcProtocol;
|
||||
return protocol ? { loaded, config: loaded.value, protocol } : undefined;
|
||||
}
|
||||
|
||||
function randomOidcValue(): string {
|
||||
return randomBytes(32).toString("base64url");
|
||||
}
|
||||
|
||||
function oidcTransactionDigest(value: string): Buffer {
|
||||
return createHash("sha256").update(value, "utf8").digest();
|
||||
}
|
||||
|
||||
function oidcTransactionMatches(value: string | undefined, expectedDigest: string): boolean {
|
||||
const canonical = typeof value === "string" && OIDC_VALUE_PATTERN.test(value);
|
||||
const expectedCanonical = /^[a-f0-9]{64}$/.test(expectedDigest);
|
||||
const supplied = oidcTransactionDigest(canonical ? value : "");
|
||||
const expected = expectedCanonical ? Buffer.from(expectedDigest, "hex") : Buffer.alloc(32);
|
||||
const matches = timingSafeEqual(supplied, expected);
|
||||
return canonical && expectedCanonical && matches;
|
||||
}
|
||||
|
||||
function oidcTransactionCookieProfile(loaded: LoadedAuthConfig): OidcTransactionCookieProfile | undefined {
|
||||
try {
|
||||
if (loaded.value.mode !== "oidc") return undefined;
|
||||
const publicUrl = parseConfiguredTransportUrl(loaded.value.publicUrl, {
|
||||
allowLoopbackHttp: true,
|
||||
originOnly: true,
|
||||
});
|
||||
if (!publicUrl) return undefined;
|
||||
if (publicUrl.protocol === "https:") return OIDC_TRANSACTION_COOKIE_PROFILES.https;
|
||||
if (publicUrl.protocol === "http:") return OIDC_TRANSACTION_COOKIE_PROFILES.loopback_http;
|
||||
return undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function oidcTransactionCookieProfileForTransport(
|
||||
transport: OidcTransactionTransport | undefined,
|
||||
): OidcTransactionCookieProfile | undefined {
|
||||
return transport === "https" || transport === "loopback_http"
|
||||
? OIDC_TRANSACTION_COOKIE_PROFILES[transport]
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function otherOidcTransactionCookieProfile(
|
||||
profile: OidcTransactionCookieProfile,
|
||||
): OidcTransactionCookieProfile {
|
||||
return profile.transport === "https"
|
||||
? OIDC_TRANSACTION_COOKIE_PROFILES.loopback_http
|
||||
: OIDC_TRANSACTION_COOKIE_PROFILES.https;
|
||||
}
|
||||
|
||||
function oidcTransactionCookieOptions(profile: OidcTransactionCookieProfile) {
|
||||
return {
|
||||
httpOnly: true,
|
||||
sameSite: "lax" as const,
|
||||
path: "/",
|
||||
secure: profile.secure,
|
||||
maxAge: OIDC_TRANSACTION_COOKIE_SECONDS,
|
||||
};
|
||||
}
|
||||
|
||||
function clearOidcTransactionCookie(reply: FastifyReply, profile: OidcTransactionCookieProfile): void {
|
||||
reply.clearCookie(profile.name, {
|
||||
httpOnly: true,
|
||||
sameSite: "lax",
|
||||
path: "/",
|
||||
secure: profile.secure,
|
||||
});
|
||||
}
|
||||
|
||||
function clearOidcTransactionCookies(reply: FastifyReply, preferred?: OidcTransactionCookieProfile): void {
|
||||
if (preferred) {
|
||||
clearOidcTransactionCookie(reply, preferred);
|
||||
clearOidcTransactionCookie(reply, otherOidcTransactionCookieProfile(preferred));
|
||||
return;
|
||||
}
|
||||
clearOidcTransactionCookie(reply, OIDC_TRANSACTION_COOKIE_PROFILES.https);
|
||||
clearOidcTransactionCookie(reply, OIDC_TRANSACTION_COOKIE_PROFILES.loopback_http);
|
||||
}
|
||||
|
||||
/**
|
||||
* Browser parsers choose one duplicate cookie value differently. Parse just our two fixed names
|
||||
* from the raw header and reject duplicates or a cross-transport sibling before hashing.
|
||||
*/
|
||||
function readOidcTransactionCookie(
|
||||
request: FastifyRequest,
|
||||
expected: OidcTransactionCookieProfile | undefined,
|
||||
): string | undefined {
|
||||
const raw = request.headers.cookie;
|
||||
if (!expected || typeof raw !== "string" || raw.length > 4096 || Array.isArray(raw)) return undefined;
|
||||
let transactionCookies = 0;
|
||||
let expectedCookies = 0;
|
||||
let expectedValue: string | undefined;
|
||||
for (const part of raw.split(";")) {
|
||||
const match = /^\s*(__Host-thothii_oidc_tx|thothii_oidc_tx)(?:=([^;]*))?\s*$/.exec(part);
|
||||
if (!match) continue;
|
||||
transactionCookies += 1;
|
||||
if (match[1] !== expected.name) continue;
|
||||
expectedCookies += 1;
|
||||
expectedValue = match[2];
|
||||
}
|
||||
return transactionCookies === 1 && expectedCookies === 1 ? expectedValue : undefined;
|
||||
}
|
||||
|
||||
function oidcCallbackUrl(
|
||||
request: FastifyRequest,
|
||||
publicUrl: string | undefined,
|
||||
): { currentUrl?: URL; state?: string } {
|
||||
if (request.url.length > MAX_OIDC_CALLBACK_QUERY_LENGTH) {
|
||||
return { state: oversizedOidcCallbackState(request.url) };
|
||||
}
|
||||
let supplied: URL;
|
||||
try {
|
||||
supplied = new URL(request.url, "http://callback.invalid");
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
if (supplied.pathname !== "/auth/oidc/callback") return {};
|
||||
const allowed = new Set(["code", "state", "error", "error_description", "error_uri", "iss"]);
|
||||
const copied = new URLSearchParams();
|
||||
let state: string | undefined;
|
||||
let valid = true;
|
||||
for (const [key, value] of supplied.searchParams) {
|
||||
if (key === "state" && state === undefined && OIDC_VALUE_PATTERN.test(value)) state = value;
|
||||
if (!allowed.has(key) || value.length > 2048 || /\p{Cc}/u.test(value) || copied.has(key)) {
|
||||
valid = false;
|
||||
continue;
|
||||
}
|
||||
copied.set(key, value);
|
||||
}
|
||||
if (!state || !valid || copied.get("state") !== state || publicUrl === undefined) return { state };
|
||||
let target: URL;
|
||||
try {
|
||||
target = new URL(OIDC_CALLBACK_PATH, publicUrl);
|
||||
} catch {
|
||||
return { state };
|
||||
}
|
||||
target.search = copied.toString();
|
||||
return { currentUrl: target, state };
|
||||
}
|
||||
|
||||
function oversizedOidcCallbackState(rawUrl: string): string | undefined {
|
||||
const prefix = "/auth/oidc/callback?";
|
||||
if (!rawUrl.startsWith(prefix)) return undefined;
|
||||
const boundedQuery = rawUrl.slice(prefix.length, MAX_OIDC_CALLBACK_QUERY_LENGTH);
|
||||
let offset = 0;
|
||||
while (offset < boundedQuery.length) {
|
||||
const separator = boundedQuery.indexOf("&", offset);
|
||||
const end = separator === -1 ? boundedQuery.length : separator;
|
||||
const parameter = boundedQuery.slice(offset, end);
|
||||
if (parameter.startsWith("state=")) {
|
||||
const value = parameter.slice("state=".length);
|
||||
if (OIDC_VALUE_PATTERN.test(value)) return value;
|
||||
}
|
||||
if (separator === -1) break;
|
||||
offset = separator + 1;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function oidcCallbackFailed(reply: FastifyReply) {
|
||||
return reply.code(401).send({ code: "oidc_callback_failed", error: "OIDC sign-in could not be completed" });
|
||||
}
|
||||
|
||||
function oidcRoles(groups: readonly string[], config: OidcAuthenticationConfig): Role[] {
|
||||
const roles = new Set<Role>();
|
||||
for (const group of groups) {
|
||||
for (const role of config.authorization.groupRoles[group] ?? []) roles.add(role);
|
||||
}
|
||||
return [...roles];
|
||||
}
|
||||
|
||||
function loginPayload(request: FastifyRequest): LoginPayload {
|
||||
const body = request.body;
|
||||
if (!body || typeof body !== "object" || Array.isArray(body)) return { username: "", password: "", remember: false };
|
||||
const input = body as Record<string, unknown>;
|
||||
return {
|
||||
username: typeof input.username === "string" && input.username.length <= MAX_USERNAME_LENGTH ? input.username : "",
|
||||
password: typeof input.password === "string" && input.password.length <= MAX_PASSWORD_LENGTH ? input.password : "",
|
||||
remember: input.remember === true,
|
||||
};
|
||||
}
|
||||
|
||||
function boundedAddress(address: string): string {
|
||||
return typeof address === "string" && address.length > 0 && address.length <= 128 ? address : "unknown";
|
||||
}
|
||||
|
||||
function argon2SafePassword(value: string): string {
|
||||
const typed = value as string & { isWellFormed?: () => boolean };
|
||||
const wellFormed = typeof typed.isWellFormed === "function"
|
||||
? typed.isWellFormed()
|
||||
: !/[\uD800-\uDFFF]/.test(value);
|
||||
const bytes = Buffer.byteLength(value, "utf8");
|
||||
if (wellFormed && bytes >= 12 && bytes <= MAX_PASSWORD_LENGTH) return value;
|
||||
// A per-attempt random value preserves the Argon2 work without turning an invalid input into
|
||||
// a reusable password that could happen to match a user's configured secret.
|
||||
return randomBytes(32).toString("base64url");
|
||||
}
|
||||
|
||||
function invalidCredentials(reply: FastifyReply): FastifyReply {
|
||||
return reply.code(401).send({ code: "invalid_credentials", error: "Invalid username or password" });
|
||||
}
|
||||
|
||||
function loginLimited(reply: FastifyReply): FastifyReply {
|
||||
return reply.code(429).send({ code: "login_rate_limited", error: "Too many login attempts" });
|
||||
}
|
||||
|
||||
function unavailable(reply: FastifyReply): FastifyReply {
|
||||
return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" });
|
||||
}
|
||||
@@ -1,618 +0,0 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
closeSync,
|
||||
constants,
|
||||
fstatSync,
|
||||
lstatSync,
|
||||
opendirSync,
|
||||
openSync,
|
||||
readSync,
|
||||
} from "node:fs";
|
||||
import type { Stats } from "node:fs";
|
||||
import { isAbsolute, join, normalize } from "node:path";
|
||||
import { parseAuthenticationConfigSource } from "./config.js";
|
||||
import { parseLocalUserRegistrySource } from "./local-registry.js";
|
||||
import type {
|
||||
AuthenticationConfigProvider,
|
||||
LoadedAuthConfig,
|
||||
LocalUserRecord,
|
||||
RuntimeProjectionSnapshot,
|
||||
} from "./types.js";
|
||||
|
||||
const MAX_AUTH_BYTES = 1 << 20;
|
||||
const MAX_USERS_BYTES = 1 << 20;
|
||||
const MAX_SELECTOR_BYTES = 4096;
|
||||
const MAX_MANIFEST_BYTES = 4096;
|
||||
const MAX_DIRECTORY_ENTRIES = 16;
|
||||
const DIR_MODE = 0o700;
|
||||
const FILE_MODE = 0o600;
|
||||
const GENERATION = /^[0-9a-f]{64}$/;
|
||||
const TRANSACTION = /^[0-9a-f]{32}$/;
|
||||
const invalid = (): Error =>
|
||||
new Error("authentication runtime projection is invalid");
|
||||
|
||||
interface Identity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
gid: number;
|
||||
mode: number;
|
||||
nlink: number;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
ctimeMs: number;
|
||||
}
|
||||
interface Selector {
|
||||
version: 1;
|
||||
state: "ready" | "blocked";
|
||||
transaction: string;
|
||||
generation?: string;
|
||||
previousGenerations?: readonly string[];
|
||||
}
|
||||
interface ManifestFile {
|
||||
name: "auth.yaml" | "users.yaml";
|
||||
size: number;
|
||||
sha256: string;
|
||||
}
|
||||
interface Manifest {
|
||||
version: 1;
|
||||
generation: string;
|
||||
mode: "local" | "oidc";
|
||||
canonicalRevision: string;
|
||||
files: readonly ManifestFile[];
|
||||
}
|
||||
class CurrentReplaced extends Error {}
|
||||
|
||||
function runtimeOwner(): number {
|
||||
if (process.platform === "win32" || typeof process.geteuid !== "function")
|
||||
throw invalid();
|
||||
const uid = process.geteuid();
|
||||
if (!Number.isSafeInteger(uid) || uid < 0) throw invalid();
|
||||
return uid;
|
||||
}
|
||||
function runtimeGroup(): number {
|
||||
if (process.platform === "win32" || typeof process.getegid !== "function")
|
||||
throw invalid();
|
||||
const gid = process.getegid();
|
||||
if (!Number.isSafeInteger(gid) || gid < 0) throw invalid();
|
||||
return gid;
|
||||
}
|
||||
function meta(info: Stats): Identity {
|
||||
return {
|
||||
dev: info.dev,
|
||||
ino: info.ino,
|
||||
uid: info.uid,
|
||||
gid: info.gid,
|
||||
mode: info.mode & 0o7777,
|
||||
nlink: info.nlink,
|
||||
size: info.size,
|
||||
mtimeMs: info.mtimeMs,
|
||||
ctimeMs: info.ctimeMs,
|
||||
};
|
||||
}
|
||||
function same(a: Identity, b: Identity): boolean {
|
||||
return (
|
||||
a.dev === b.dev &&
|
||||
a.ino === b.ino &&
|
||||
a.uid === b.uid &&
|
||||
a.gid === b.gid &&
|
||||
a.mode === b.mode &&
|
||||
a.nlink === b.nlink &&
|
||||
a.size === b.size &&
|
||||
a.mtimeMs === b.mtimeMs &&
|
||||
a.ctimeMs === b.ctimeMs
|
||||
);
|
||||
}
|
||||
function directory(info: Stats, uid: number): Identity {
|
||||
const value = meta(info);
|
||||
if (
|
||||
!info.isDirectory() ||
|
||||
value.uid !== uid ||
|
||||
value.gid !== runtimeGroup() ||
|
||||
value.mode !== DIR_MODE
|
||||
)
|
||||
throw invalid();
|
||||
return value;
|
||||
}
|
||||
function regular(info: Stats, uid: number, maximum: number): Identity {
|
||||
const value = meta(info);
|
||||
if (
|
||||
!info.isFile() ||
|
||||
value.uid !== uid ||
|
||||
value.gid !== runtimeGroup() ||
|
||||
value.mode !== FILE_MODE ||
|
||||
value.nlink !== 1 ||
|
||||
value.size < 0 ||
|
||||
value.size > maximum
|
||||
)
|
||||
throw invalid();
|
||||
return value;
|
||||
}
|
||||
function checkRoot(root: string): void {
|
||||
if (
|
||||
typeof root !== "string" ||
|
||||
root.length === 0 ||
|
||||
root.includes("\0") ||
|
||||
!isAbsolute(root) ||
|
||||
normalize(root) !== root ||
|
||||
(root.length > 1 && root.endsWith("/"))
|
||||
)
|
||||
throw invalid();
|
||||
}
|
||||
function openDirectory(
|
||||
path: string,
|
||||
uid: number,
|
||||
): { fd: number; identity: Identity } {
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
const before = directory(lstatSync(path) as Stats, uid);
|
||||
fd = openSync(
|
||||
path,
|
||||
constants.O_RDONLY |
|
||||
(constants.O_DIRECTORY ?? 0) |
|
||||
constants.O_NOFOLLOW |
|
||||
constants.O_NONBLOCK,
|
||||
);
|
||||
const opened = directory(fstatSync(fd) as Stats, uid);
|
||||
if (!same(before, opened)) throw invalid();
|
||||
return { fd, identity: opened };
|
||||
} catch {
|
||||
if (fd !== undefined)
|
||||
try {
|
||||
closeSync(fd);
|
||||
} catch {}
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
function stableDirectory(
|
||||
path: string,
|
||||
opened: { fd: number; identity: Identity },
|
||||
uid: number,
|
||||
): void {
|
||||
if (
|
||||
!same(opened.identity, directory(fstatSync(opened.fd) as Stats, uid)) ||
|
||||
!same(opened.identity, directory(lstatSync(path) as Stats, uid))
|
||||
)
|
||||
throw invalid();
|
||||
}
|
||||
function entries(path: string, uid: number, expected: readonly string[]): void {
|
||||
const opened = openDirectory(path, uid);
|
||||
try {
|
||||
const directory = opendirSync(`/proc/self/fd/${opened.fd}`, {
|
||||
bufferSize: 1,
|
||||
});
|
||||
const names: string[] = [];
|
||||
try {
|
||||
for (;;) {
|
||||
const entry = directory.readSync();
|
||||
if (entry === null) break;
|
||||
if (names.length === MAX_DIRECTORY_ENTRIES) throw invalid();
|
||||
names.push(entry.name);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
directory.closeSync();
|
||||
} catch {}
|
||||
}
|
||||
if (
|
||||
names.length !== expected.length ||
|
||||
new Set(names).size !== names.length ||
|
||||
names.some((name) => !expected.includes(name))
|
||||
)
|
||||
throw invalid();
|
||||
stableDirectory(path, opened, uid);
|
||||
} finally {
|
||||
try {
|
||||
closeSync(opened.fd);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function replaced(before: Identity, after: Identity): boolean {
|
||||
return before.dev !== after.dev || before.ino !== after.ino;
|
||||
}
|
||||
interface RootObservation {
|
||||
descriptor: Identity;
|
||||
path: Identity;
|
||||
}
|
||||
function sameObject(left: Identity, right: Identity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino;
|
||||
}
|
||||
function observeRootAtPathAndDescriptor(
|
||||
root: string,
|
||||
openedRoot: { fd: number; identity: Identity },
|
||||
uid: number,
|
||||
): RootObservation {
|
||||
const openedAfter = directory(fstatSync(openedRoot.fd) as Stats, uid);
|
||||
const pathAfter = directory(lstatSync(root) as Stats, uid);
|
||||
if (!sameObject(openedAfter, pathAfter)) throw invalid();
|
||||
return { descriptor: openedAfter, path: pathAfter };
|
||||
}
|
||||
function validateRootAndCurrentAfterLoad(
|
||||
root: string,
|
||||
openedRoot: { fd: number; identity: Identity },
|
||||
currentPath: string,
|
||||
selectedCurrent: Identity,
|
||||
uid: number,
|
||||
): void {
|
||||
const rootBeforeCurrent = observeRootAtPathAndDescriptor(
|
||||
root,
|
||||
openedRoot,
|
||||
uid,
|
||||
);
|
||||
const currentAfter = regular(
|
||||
lstatSync(currentPath) as Stats,
|
||||
uid,
|
||||
MAX_SELECTOR_BYTES,
|
||||
);
|
||||
const rootAfterCurrent = observeRootAtPathAndDescriptor(
|
||||
root,
|
||||
openedRoot,
|
||||
uid,
|
||||
);
|
||||
if (
|
||||
!same(openedRoot.identity, rootBeforeCurrent.descriptor) ||
|
||||
!same(rootBeforeCurrent.descriptor, rootBeforeCurrent.path) ||
|
||||
!same(rootBeforeCurrent.path, rootAfterCurrent.descriptor) ||
|
||||
!same(rootAfterCurrent.descriptor, rootAfterCurrent.path)
|
||||
) {
|
||||
const latestCurrent = regular(
|
||||
lstatSync(currentPath) as Stats,
|
||||
uid,
|
||||
MAX_SELECTOR_BYTES,
|
||||
);
|
||||
if (replaced(selectedCurrent, latestCurrent)) throw new CurrentReplaced();
|
||||
throw invalid();
|
||||
}
|
||||
if (!same(selectedCurrent, currentAfter)) {
|
||||
if (replaced(selectedCurrent, currentAfter)) throw new CurrentReplaced();
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
function readRegular(
|
||||
path: string,
|
||||
parentPath: string,
|
||||
uid: number,
|
||||
maximum: number,
|
||||
retryOnReplacement = false,
|
||||
): { bytes: Buffer; identity: Identity } {
|
||||
let fd: number | undefined;
|
||||
let parent: { fd: number; identity: Identity } | undefined;
|
||||
try {
|
||||
parent = openDirectory(parentPath, uid);
|
||||
const before = regular(lstatSync(path) as Stats, uid, maximum);
|
||||
fd = openSync(
|
||||
path,
|
||||
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
|
||||
);
|
||||
const opened = regular(fstatSync(fd) as Stats, uid, maximum);
|
||||
if (!same(before, opened)) {
|
||||
if (retryOnReplacement && replaced(before, opened))
|
||||
throw new CurrentReplaced();
|
||||
throw invalid();
|
||||
}
|
||||
const buffer = Buffer.allocUnsafe(maximum + 1);
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const count = readSync(fd, buffer, offset, buffer.length - offset, null);
|
||||
if (count === 0) break;
|
||||
offset += count;
|
||||
}
|
||||
if (offset > maximum) throw invalid();
|
||||
const after = regular(fstatSync(fd) as Stats, uid, maximum);
|
||||
const atPath = regular(lstatSync(path) as Stats, uid, maximum);
|
||||
if (!same(opened, after) || !same(after, atPath)) {
|
||||
if (retryOnReplacement && replaced(after, atPath))
|
||||
throw new CurrentReplaced();
|
||||
throw invalid();
|
||||
}
|
||||
stableDirectory(parentPath, parent, uid);
|
||||
return { bytes: buffer.subarray(0, offset), identity: after };
|
||||
} catch (error) {
|
||||
if (error instanceof CurrentReplaced) throw error;
|
||||
throw invalid();
|
||||
} finally {
|
||||
if (fd !== undefined)
|
||||
try {
|
||||
closeSync(fd);
|
||||
} catch {}
|
||||
if (parent)
|
||||
try {
|
||||
closeSync(parent.fd);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function text(bytes: Buffer): string {
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
function object(value: unknown): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value))
|
||||
throw invalid();
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
function safeGeneration(value: unknown): string {
|
||||
if (typeof value !== "string" || !GENERATION.test(value)) throw invalid();
|
||||
return value;
|
||||
}
|
||||
function strictJson<T>(
|
||||
contents: string,
|
||||
normalizeValue: (raw: unknown) => T,
|
||||
): T {
|
||||
try {
|
||||
const value = normalizeValue(JSON.parse(contents));
|
||||
if (`${JSON.stringify(value)}\n` !== contents) throw invalid();
|
||||
return value;
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
function selector(contents: string): Selector {
|
||||
return strictJson(contents, (raw) => {
|
||||
const value = object(raw);
|
||||
if (
|
||||
value.version !== 1 ||
|
||||
typeof value.transaction !== "string" ||
|
||||
!TRANSACTION.test(value.transaction)
|
||||
)
|
||||
throw invalid();
|
||||
if (value.state === "blocked" && Object.keys(value).length === 3)
|
||||
return { version: 1, state: "blocked", transaction: value.transaction };
|
||||
if (
|
||||
value.state !== "ready" ||
|
||||
(Object.keys(value).length !== 4 && Object.keys(value).length !== 5)
|
||||
)
|
||||
throw invalid();
|
||||
const generation = safeGeneration(value.generation);
|
||||
const previousGenerations =
|
||||
value.previousGenerations === undefined
|
||||
? []
|
||||
: Array.isArray(value.previousGenerations)
|
||||
? value.previousGenerations.map(safeGeneration)
|
||||
: (() => {
|
||||
throw invalid();
|
||||
})();
|
||||
if (
|
||||
previousGenerations.length > 2 ||
|
||||
(previousGenerations.length === 0 && Object.keys(value).length !== 4) ||
|
||||
(previousGenerations.length > 0 && Object.keys(value).length !== 5)
|
||||
)
|
||||
throw invalid();
|
||||
if (
|
||||
new Set([generation, ...previousGenerations]).size !==
|
||||
previousGenerations.length + 1
|
||||
)
|
||||
throw invalid();
|
||||
return {
|
||||
version: 1,
|
||||
state: "ready",
|
||||
transaction: value.transaction,
|
||||
generation,
|
||||
...(previousGenerations.length > 0 ? { previousGenerations } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
function manifest(contents: string): Manifest {
|
||||
return strictJson(contents, (raw) => {
|
||||
const value = object(raw);
|
||||
if (
|
||||
Object.keys(value).length !== 5 ||
|
||||
value.version !== 1 ||
|
||||
(value.mode !== "local" && value.mode !== "oidc")
|
||||
)
|
||||
throw invalid();
|
||||
const mode = value.mode;
|
||||
const generation = safeGeneration(value.generation);
|
||||
if (
|
||||
value.canonicalRevision !== `sha256:${generation}` ||
|
||||
!Array.isArray(value.files)
|
||||
)
|
||||
throw invalid();
|
||||
const wanted: readonly ("auth.yaml" | "users.yaml")[] =
|
||||
mode === "local" ? ["auth.yaml", "users.yaml"] : ["auth.yaml"];
|
||||
if (value.files.length !== wanted.length) throw invalid();
|
||||
const files: ManifestFile[] = value.files.map((candidate, index) => {
|
||||
const item = object(candidate);
|
||||
const name = wanted[index]!;
|
||||
const maximum = name === "auth.yaml" ? MAX_AUTH_BYTES : MAX_USERS_BYTES;
|
||||
if (
|
||||
Object.keys(item).length !== 3 ||
|
||||
item.name !== name ||
|
||||
!Number.isSafeInteger(item.size) ||
|
||||
(item.size as number) < 0 ||
|
||||
(item.size as number) > maximum ||
|
||||
typeof item.sha256 !== "string" ||
|
||||
!GENERATION.test(item.sha256)
|
||||
)
|
||||
throw invalid();
|
||||
return { name, size: item.size as number, sha256: item.sha256 };
|
||||
});
|
||||
return {
|
||||
version: 1,
|
||||
generation,
|
||||
mode,
|
||||
canonicalRevision: value.canonicalRevision as string,
|
||||
files,
|
||||
};
|
||||
});
|
||||
}
|
||||
function digest(bytes: Buffer): string {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
function generationFor(
|
||||
mode: "local" | "oidc",
|
||||
auth: Buffer,
|
||||
users?: Buffer,
|
||||
): string {
|
||||
return digest(
|
||||
Buffer.from(
|
||||
`thothii-auth-projection-v1\nmode=${mode}\nauth=${digest(auth)}\nusers=${mode === "local" && users ? digest(users) : "-"}\n`,
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
}
|
||||
function snapshot(
|
||||
generation: string,
|
||||
users?: readonly LocalUserRecord[],
|
||||
): RuntimeProjectionSnapshot {
|
||||
const localUsers =
|
||||
users === undefined
|
||||
? undefined
|
||||
: Object.freeze(
|
||||
users.map((user) =>
|
||||
Object.freeze({ ...user, roles: Object.freeze([...user.roles]) }),
|
||||
),
|
||||
);
|
||||
return Object.freeze({
|
||||
generation,
|
||||
canonicalRevision: `sha256:${generation}`,
|
||||
...(localUsers ? { localUsers } : {}),
|
||||
});
|
||||
}
|
||||
interface ValidGeneration {
|
||||
value: ReturnType<typeof parseAuthenticationConfigSource>;
|
||||
users?: readonly LocalUserRecord[];
|
||||
}
|
||||
function validateGeneration(
|
||||
generationsPath: string,
|
||||
generation: string,
|
||||
uid: number,
|
||||
): ValidGeneration {
|
||||
const selectedPath = join(generationsPath, generation);
|
||||
const openedGeneration = openDirectory(selectedPath, uid);
|
||||
try {
|
||||
const readManifest = readRegular(
|
||||
join(selectedPath, "manifest.json"),
|
||||
selectedPath,
|
||||
uid,
|
||||
MAX_MANIFEST_BYTES,
|
||||
);
|
||||
const loadedManifest = manifest(text(readManifest.bytes));
|
||||
if (loadedManifest.generation !== generation) throw invalid();
|
||||
entries(
|
||||
selectedPath,
|
||||
uid,
|
||||
[
|
||||
...loadedManifest.files.map((item) => item.name),
|
||||
"manifest.json",
|
||||
].sort(),
|
||||
);
|
||||
const auth = readRegular(
|
||||
join(selectedPath, "auth.yaml"),
|
||||
selectedPath,
|
||||
uid,
|
||||
MAX_AUTH_BYTES,
|
||||
);
|
||||
if (
|
||||
loadedManifest.files[0]?.size !== auth.bytes.length ||
|
||||
loadedManifest.files[0]?.sha256 !== digest(auth.bytes)
|
||||
)
|
||||
throw invalid();
|
||||
const value = parseAuthenticationConfigSource(text(auth.bytes));
|
||||
if (value.mode !== loadedManifest.mode) throw invalid();
|
||||
let users: readonly LocalUserRecord[] | undefined;
|
||||
let userBytes: Buffer | undefined;
|
||||
if (loadedManifest.mode === "local") {
|
||||
const readUsers = readRegular(
|
||||
join(selectedPath, "users.yaml"),
|
||||
selectedPath,
|
||||
uid,
|
||||
MAX_USERS_BYTES,
|
||||
);
|
||||
if (
|
||||
loadedManifest.files[1]?.size !== readUsers.bytes.length ||
|
||||
loadedManifest.files[1]?.sha256 !== digest(readUsers.bytes)
|
||||
)
|
||||
throw invalid();
|
||||
userBytes = readUsers.bytes;
|
||||
users = parseLocalUserRegistrySource(text(userBytes));
|
||||
}
|
||||
if (
|
||||
generationFor(loadedManifest.mode, auth.bytes, userBytes) !== generation
|
||||
)
|
||||
throw invalid();
|
||||
stableDirectory(selectedPath, openedGeneration, uid);
|
||||
return { value, users };
|
||||
} finally {
|
||||
try {
|
||||
closeSync(openedGeneration.fd);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function load(root: string): LoadedAuthConfig {
|
||||
const uid = runtimeOwner();
|
||||
checkRoot(root);
|
||||
const openedRoot = openDirectory(root, uid);
|
||||
try {
|
||||
entries(root, uid, ["CURRENT", "generations"]);
|
||||
const currentPath = join(root, "CURRENT");
|
||||
const selectedCurrent = readRegular(
|
||||
currentPath,
|
||||
root,
|
||||
uid,
|
||||
MAX_SELECTOR_BYTES,
|
||||
true,
|
||||
);
|
||||
const selected = selector(text(selectedCurrent.bytes));
|
||||
if (selected.state !== "ready" || !selected.generation) throw invalid();
|
||||
const generationsPath = join(root, "generations");
|
||||
const openedGenerations = openDirectory(generationsPath, uid);
|
||||
try {
|
||||
entries(generationsPath, uid, [
|
||||
selected.generation,
|
||||
...(selected.previousGenerations ?? []),
|
||||
]);
|
||||
const selectedGeneration = validateGeneration(
|
||||
generationsPath,
|
||||
selected.generation,
|
||||
uid,
|
||||
);
|
||||
for (const predecessor of selected.previousGenerations ?? [])
|
||||
validateGeneration(generationsPath, predecessor, uid);
|
||||
stableDirectory(generationsPath, openedGenerations, uid);
|
||||
validateRootAndCurrentAfterLoad(
|
||||
root,
|
||||
openedRoot,
|
||||
currentPath,
|
||||
selectedCurrent.identity,
|
||||
uid,
|
||||
);
|
||||
return {
|
||||
value: selectedGeneration.value,
|
||||
revision: selected.generation,
|
||||
sourcePath: join(generationsPath, selected.generation, "auth.yaml"),
|
||||
runtimeProjection: snapshot(
|
||||
selected.generation,
|
||||
selectedGeneration.users,
|
||||
),
|
||||
};
|
||||
} finally {
|
||||
try {
|
||||
closeSync(openedGenerations.fd);
|
||||
} catch {}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
closeSync(openedRoot.fd);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
export function createProjectedAuthenticationConfigProvider(
|
||||
root: string,
|
||||
): AuthenticationConfigProvider {
|
||||
return {
|
||||
current(): LoadedAuthConfig {
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
return load(root);
|
||||
} catch (error) {
|
||||
if (error instanceof CurrentReplaced && attempt === 0) continue;
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
throw invalid();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
export const AUTHENTICATION_SECRET_LIMITS = Object.freeze({
|
||||
THT_OIDC_CLIENT_SECRET: 4096,
|
||||
THT_AUTHENTIK_API_TOKEN: 16 * 1024,
|
||||
} as const);
|
||||
|
||||
export type AuthenticationSecretReference = keyof typeof AUTHENTICATION_SECRET_LIMITS;
|
||||
|
||||
export function isAuthenticationSecretReference(value: string): value is AuthenticationSecretReference {
|
||||
return Object.prototype.hasOwnProperty.call(AUTHENTICATION_SECRET_LIMITS, value);
|
||||
}
|
||||
|
||||
/** One policy shared by bundle loading, runtime adapters, and static diagnostics. */
|
||||
export function isUsableAuthenticationSecret(
|
||||
name: AuthenticationSecretReference,
|
||||
value: unknown,
|
||||
): value is string {
|
||||
return typeof value === "string" && value.length > 0
|
||||
&& value.length <= AUTHENTICATION_SECRET_LIMITS[name] && !/\p{Cc}/u.test(value);
|
||||
}
|
||||
@@ -1,754 +0,0 @@
|
||||
import { createHash, hkdfSync, randomBytes } from "node:crypto";
|
||||
import { z } from "zod";
|
||||
import type { PrincipalContext } from "./principal.js";
|
||||
import type {
|
||||
AuthSessionRecord,
|
||||
OidcStateRecord,
|
||||
OidcTransactionTransport,
|
||||
Permission,
|
||||
Role,
|
||||
} from "./types.js";
|
||||
import {
|
||||
createPosixAuthStorageBridge,
|
||||
createWindowsAuthStorageBridge,
|
||||
type WindowsAuthStorageBridge,
|
||||
} from "./windows-auth-storage.js";
|
||||
|
||||
const TOKEN_BYTES = 32;
|
||||
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/;
|
||||
const DIGEST_FILENAME_PATTERN = /^[a-f0-9]{64}\.json$/;
|
||||
const CLAIM_FILENAME_PATTERN = /^[a-f0-9]{64}\.claim$/;
|
||||
const OIDC_SLOT_FILENAME_PATTERN = /^slot-(\d{2})\.json$/;
|
||||
const MAX_SESSION_RECORD_BYTES = 16 * 1024;
|
||||
const MAX_OIDC_STATE_RECORD_BYTES = 8 * 1024;
|
||||
const MAX_OIDC_SLOT_RECORD_BYTES = 512;
|
||||
const MAX_TTL_MS = 365 * 24 * 60 * 60 * 1000;
|
||||
const OIDC_STATE_TTL_MS = 10 * 60 * 1000;
|
||||
const OIDC_STATE_CAPACITY = 64;
|
||||
const MAX_OIDC_STORAGE_ENTRIES = OIDC_STATE_CAPACITY * 3;
|
||||
const MAX_SESSION_PRUNE_ENTRIES = 512;
|
||||
const TOUCH_INTERVAL_MS = 5 * 60 * 1000;
|
||||
const CSRF_CONTEXT = Buffer.from("thothii-csrf-v1", "utf8");
|
||||
const EMPTY_HKDF_SALT = Buffer.alloc(0);
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
const PERMISSIONS = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
] as const satisfies readonly Permission[];
|
||||
|
||||
const invalid = (): Error => new Error("auth_session_store_invalid");
|
||||
|
||||
export interface SessionCreateInput {
|
||||
principal: PrincipalContext;
|
||||
method: "local" | "oidc" | "upstream";
|
||||
remembered: boolean;
|
||||
userAuthRevision?: number;
|
||||
authConfigRevision: string;
|
||||
idleTtlMs: number;
|
||||
absoluteTtlMs: number;
|
||||
}
|
||||
|
||||
export interface CreatedAuthSession {
|
||||
token: string;
|
||||
csrfToken: string;
|
||||
record: AuthSessionRecord;
|
||||
}
|
||||
|
||||
export interface OidcStateCreateInput {
|
||||
nonce: string;
|
||||
codeVerifier: string;
|
||||
returnTo: "/";
|
||||
authConfigRevision: string;
|
||||
issuer: string;
|
||||
browserTransactionDigest: string;
|
||||
browserTransactionTransport: OidcTransactionTransport;
|
||||
}
|
||||
|
||||
export interface CreatedOidcState {
|
||||
state: string;
|
||||
record: OidcStateRecord;
|
||||
}
|
||||
|
||||
export interface LocalSessionUser {
|
||||
enabled: boolean;
|
||||
authRevision: number;
|
||||
roles: readonly Role[];
|
||||
}
|
||||
|
||||
export interface CurrentLocalSessionUser {
|
||||
revision: string;
|
||||
user: LocalSessionUser | undefined;
|
||||
}
|
||||
|
||||
/** Operational validity-source failures must not masquerade as revoked credentials. */
|
||||
export class AuthSessionOperationalError extends Error {
|
||||
constructor() {
|
||||
super("auth_session_operational_error");
|
||||
}
|
||||
}
|
||||
|
||||
export class OidcStateCapacityError extends Error {
|
||||
constructor() {
|
||||
super("auth_oidc_state_capacity");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The route layer supplies the current installation revision and local-registry lookup.
|
||||
* Supplying this hook makes every resolve an authorization-generation check.
|
||||
*/
|
||||
export interface AuthSessionValidity {
|
||||
currentAuthConfigRevision(): string | Promise<string>;
|
||||
findLocalUser?(subject: string): LocalSessionUser | undefined | Promise<LocalSessionUser | undefined>;
|
||||
currentLocalUser?(subject: string): CurrentLocalSessionUser | Promise<CurrentLocalSessionUser>;
|
||||
}
|
||||
|
||||
export interface AuthSessionStore {
|
||||
create(input: SessionCreateInput, now?: Date): Promise<CreatedAuthSession>;
|
||||
resolve(token: string, now?: Date, validity?: AuthSessionValidity): Promise<AuthSessionRecord | undefined>;
|
||||
touch(token: string, now?: Date): Promise<void>;
|
||||
revoke(token: string): Promise<void>;
|
||||
prune(now?: Date): Promise<number>;
|
||||
createOidcState(input: OidcStateCreateInput, now?: Date): Promise<CreatedOidcState>;
|
||||
consumeOidcState(state: string, now?: Date): Promise<OidcStateRecord | undefined>;
|
||||
}
|
||||
|
||||
/** Narrow test seams for the native tht-backed storage adaptors. */
|
||||
export interface FileAuthSessionStoreOptions {
|
||||
windowsStorageBridge?: WindowsAuthStorageBridge;
|
||||
/** Test seam; production uses the bounded hidden tht bridge for every POSIX record operation. */
|
||||
posixStorageBridge?: WindowsAuthStorageBridge;
|
||||
/** Test-only capacity seam; production always uses the fixed 64-state bound. */
|
||||
oidcStateCapacity?: number;
|
||||
}
|
||||
|
||||
interface SessionDirectoryPage {
|
||||
entries: string[];
|
||||
more: boolean;
|
||||
}
|
||||
|
||||
const text = z.string().min(1).max(512).refine((value) => !/[\u0000-\u001f\u007f]/.test(value));
|
||||
const timestamp = z.string().length(24).refine((value) => {
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) && new Date(parsed).toISOString() === value;
|
||||
});
|
||||
const role = z.enum(ROLES);
|
||||
const permission = z.enum(PERMISSIONS);
|
||||
const distinct = <T>(items: readonly T[]): boolean => new Set(items).size === items.length;
|
||||
const sessionRecordSchema = z.strictObject({
|
||||
version: z.literal(1),
|
||||
issuer: text,
|
||||
subject: text,
|
||||
displayName: text.optional(),
|
||||
method: z.enum(["local", "oidc", "upstream"]),
|
||||
roles: z.array(role).max(ROLES.length).refine(distinct),
|
||||
permissions: z.array(permission).max(PERMISSIONS.length).refine(distinct),
|
||||
userAuthRevision: z.number().int().positive().safe().optional(),
|
||||
authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
remembered: z.boolean(),
|
||||
createdAt: timestamp,
|
||||
lastSeenAt: timestamp,
|
||||
idleExpiresAt: timestamp,
|
||||
absoluteExpiresAt: timestamp,
|
||||
}).superRefine((record, context) => {
|
||||
const createdAt = Date.parse(record.createdAt);
|
||||
const lastSeenAt = Date.parse(record.lastSeenAt);
|
||||
const idleExpiresAt = Date.parse(record.idleExpiresAt);
|
||||
const absoluteExpiresAt = Date.parse(record.absoluteExpiresAt);
|
||||
if (lastSeenAt < createdAt || idleExpiresAt < lastSeenAt || idleExpiresAt > absoluteExpiresAt
|
||||
|| absoluteExpiresAt < createdAt || absoluteExpiresAt - createdAt > MAX_TTL_MS) {
|
||||
context.addIssue({ code: "custom", message: "invalid session lifetime" });
|
||||
}
|
||||
if (record.method === "local" && record.userAuthRevision === undefined) {
|
||||
context.addIssue({ code: "custom", message: "local revision is required" });
|
||||
}
|
||||
if (record.method !== "local" && record.userAuthRevision !== undefined) {
|
||||
context.addIssue({ code: "custom", message: "non-local revision is forbidden" });
|
||||
}
|
||||
});
|
||||
const oidcStateRecordSchema = z.strictObject({
|
||||
version: z.literal(1),
|
||||
nonce: z.string().min(16).max(512).regex(/^[A-Za-z0-9_-]+$/),
|
||||
codeVerifier: z.string().min(43).max(128).regex(/^[A-Za-z0-9._~-]+$/),
|
||||
returnTo: z.literal("/"),
|
||||
authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
issuer: z.string().min(1).max(2048).refine((value) => !/\p{Cc}/u.test(value)),
|
||||
browserTransactionDigest: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
// Existing ten-minute records from before this field was introduced can be consumed and
|
||||
// rejected by the route. New records always receive the required input field below.
|
||||
browserTransactionTransport: z.enum(["https", "loopback_http"]).optional(),
|
||||
capacitySlot: z.number().int().min(0).max(OIDC_STATE_CAPACITY - 1).optional(),
|
||||
createdAt: timestamp,
|
||||
expiresAt: timestamp,
|
||||
}).superRefine((record, context) => {
|
||||
const lifetime = Date.parse(record.expiresAt) - Date.parse(record.createdAt);
|
||||
if (lifetime <= 0 || lifetime > OIDC_STATE_TTL_MS) {
|
||||
context.addIssue({ code: "custom", message: "invalid OIDC state lifetime" });
|
||||
}
|
||||
});
|
||||
const oidcSlotRecordSchema = z.strictObject({
|
||||
version: z.literal(1),
|
||||
stateFilename: z.string().regex(DIGEST_FILENAME_PATTERN),
|
||||
expiresAt: timestamp,
|
||||
});
|
||||
const sessionInputSchema = z.strictObject({
|
||||
principal: z.strictObject({
|
||||
issuer: text,
|
||||
subject: text,
|
||||
displayName: text.optional(),
|
||||
roles: z.array(role).max(ROLES.length).refine(distinct),
|
||||
permissions: z.array(permission).max(PERMISSIONS.length).refine(distinct),
|
||||
isAdmin: z.boolean(),
|
||||
}),
|
||||
method: z.enum(["local", "oidc", "upstream"]),
|
||||
remembered: z.boolean(),
|
||||
userAuthRevision: z.number().int().positive().safe().optional(),
|
||||
authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
idleTtlMs: z.number().int().min(1).max(MAX_TTL_MS),
|
||||
absoluteTtlMs: z.number().int().min(1).max(MAX_TTL_MS),
|
||||
}).superRefine((input, context) => {
|
||||
if (input.principal.isAdmin !== input.principal.roles.includes("admin")) {
|
||||
context.addIssue({ code: "custom", message: "principal roles disagree" });
|
||||
}
|
||||
if (input.method === "local" && input.userAuthRevision === undefined) {
|
||||
context.addIssue({ code: "custom", message: "local revision is required" });
|
||||
}
|
||||
if (input.method !== "local" && input.userAuthRevision !== undefined) {
|
||||
context.addIssue({ code: "custom", message: "non-local revision is forbidden" });
|
||||
}
|
||||
});
|
||||
const oidcStateInputSchema = z.strictObject({
|
||||
nonce: z.string().min(16).max(512).regex(/^[A-Za-z0-9_-]+$/),
|
||||
codeVerifier: z.string().min(43).max(128).regex(/^[A-Za-z0-9._~-]+$/),
|
||||
returnTo: z.literal("/"),
|
||||
authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
issuer: z.string().min(1).max(2048).refine((value) => !/\p{Cc}/u.test(value)),
|
||||
browserTransactionDigest: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
browserTransactionTransport: z.enum(["https", "loopback_http"]),
|
||||
});
|
||||
|
||||
function canonicalRawValue(value: string): boolean {
|
||||
if (typeof value !== "string" || !TOKEN_PATTERN.test(value)) return false;
|
||||
try {
|
||||
const bytes = Buffer.from(value, "base64url");
|
||||
return bytes.length === TOKEN_BYTES && bytes.toString("base64url") === value;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function digestFilename(rawValue: string): string {
|
||||
return `${createHash("sha256").update(rawValue).digest("hex")}.json`;
|
||||
}
|
||||
|
||||
function claimFilename(filename: string): string {
|
||||
if (!DIGEST_FILENAME_PATTERN.test(filename)) throw invalid();
|
||||
return filename.slice(0, -".json".length) + ".claim";
|
||||
}
|
||||
|
||||
function oidcSlotFilename(index: number): string {
|
||||
if (!Number.isInteger(index) || index < 0 || index >= OIDC_STATE_CAPACITY) throw invalid();
|
||||
return `slot-${String(index).padStart(2, "0")}.json`;
|
||||
}
|
||||
|
||||
function oidcSlotIndex(filename: string): number | undefined {
|
||||
const match = OIDC_SLOT_FILENAME_PATTERN.exec(filename);
|
||||
if (!match) return undefined;
|
||||
const index = Number(match[1]);
|
||||
return Number.isInteger(index) && index >= 0 && index < OIDC_STATE_CAPACITY ? index : undefined;
|
||||
}
|
||||
|
||||
function parseSessionRecord(source: string): AuthSessionRecord {
|
||||
try {
|
||||
return sessionRecordSchema.parse(JSON.parse(source)) as AuthSessionRecord;
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function parseOidcStateRecord(source: string): OidcStateRecord {
|
||||
try {
|
||||
return oidcStateRecordSchema.parse(JSON.parse(source)) as OidcStateRecord;
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
type OidcSlotRecord = z.infer<typeof oidcSlotRecordSchema>;
|
||||
|
||||
function parseOidcSlotRecord(source: string): OidcSlotRecord {
|
||||
try {
|
||||
return oidcSlotRecordSchema.parse(JSON.parse(source));
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function parseWindowsRecord<T>(contents: Buffer, maximumBytes: number, parse: (source: string) => T): T {
|
||||
if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > maximumBytes) throw invalid();
|
||||
try {
|
||||
return parse(new TextDecoder("utf-8", { fatal: true }).decode(contents));
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
interface StoredOidcSlot {
|
||||
filename: string;
|
||||
index: number;
|
||||
record: OidcSlotRecord;
|
||||
}
|
||||
|
||||
function serialize(record: AuthSessionRecord | OidcStateRecord | OidcSlotRecord, maximumBytes: number): Buffer {
|
||||
const contents = Buffer.from(`${JSON.stringify(record)}\n`, "utf8");
|
||||
if (contents.length > maximumBytes) throw invalid();
|
||||
return contents;
|
||||
}
|
||||
|
||||
function dateMilliseconds(now: Date): number {
|
||||
if (!(now instanceof Date) || !Number.isFinite(now.getTime())) throw invalid();
|
||||
return now.getTime();
|
||||
}
|
||||
|
||||
function isoAt(milliseconds: number): string {
|
||||
if (!Number.isSafeInteger(milliseconds) || !Number.isFinite(milliseconds)) throw invalid();
|
||||
try {
|
||||
return new Date(milliseconds).toISOString();
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function sessionExpired(record: AuthSessionRecord, nowMs: number): boolean {
|
||||
return nowMs >= Date.parse(record.idleExpiresAt) || nowMs >= Date.parse(record.absoluteExpiresAt);
|
||||
}
|
||||
|
||||
function oidcStateExpired(record: OidcStateRecord, nowMs: number): boolean {
|
||||
return nowMs >= Date.parse(record.expiresAt);
|
||||
}
|
||||
|
||||
function equalRoleSets(left: readonly Role[], right: readonly Role[]): boolean {
|
||||
if (!distinct(left) || !distinct(right) || left.length !== right.length) return false;
|
||||
if (!left.every((value) => ROLES.includes(value)) || !right.every((value) => ROLES.includes(value))) return false;
|
||||
return [...left].sort().every((value, index) => value === [...right].sort()[index]);
|
||||
}
|
||||
|
||||
function validLocalUser(user: LocalSessionUser | undefined, record: AuthSessionRecord): boolean {
|
||||
return user !== undefined && user.enabled === true && Number.isSafeInteger(user.authRevision)
|
||||
&& user.authRevision > 0 && user.authRevision === record.userAuthRevision
|
||||
&& equalRoleSets(user.roles, record.roles);
|
||||
}
|
||||
|
||||
async function recordIsCurrent(record: AuthSessionRecord, validity: AuthSessionValidity | undefined): Promise<boolean> {
|
||||
// A root-only store remains useful for creation/diagnostics, but is intentionally incapable
|
||||
// of authenticating a principal. Task 8 must supply config and local-registry dependencies.
|
||||
if (!validity) return false;
|
||||
if (record.method === "local" && validity.currentLocalUser) {
|
||||
const current = await validity.currentLocalUser(record.subject);
|
||||
return typeof current.revision === "string" && current.revision === record.authConfigRevision
|
||||
&& validLocalUser(current.user, record);
|
||||
}
|
||||
const revision = await validity.currentAuthConfigRevision();
|
||||
if (typeof revision !== "string" || revision !== record.authConfigRevision) return false;
|
||||
if (record.method !== "local") return true;
|
||||
return validity.findLocalUser === undefined ? false : validLocalUser(await validity.findLocalUser(record.subject), record);
|
||||
}
|
||||
|
||||
const locks = new Map<string, Promise<void>>();
|
||||
|
||||
async function withLock<T>(key: string, operation: () => Promise<T>): Promise<T> {
|
||||
const previous = locks.get(key) ?? Promise.resolve();
|
||||
let release: (() => void) | undefined;
|
||||
const current = new Promise<void>((resolve) => { release = resolve; });
|
||||
locks.set(key, current);
|
||||
await previous;
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release?.();
|
||||
if (locks.get(key) === current) locks.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function lockKey(root: string, directory: "sessions" | "oidc", filename: string): string {
|
||||
return `${root}\0${directory}\0${filename}`;
|
||||
}
|
||||
|
||||
/** Derive a one-way, domain-separated 256-bit CSRF value without persisting it. */
|
||||
export function deriveCsrfToken(sessionToken: string): string {
|
||||
if (!canonicalRawValue(sessionToken)) throw invalid();
|
||||
try {
|
||||
const sessionBytes = Buffer.from(sessionToken, "base64url");
|
||||
return Buffer.from(hkdfSync("sha256", sessionBytes, EMPTY_HKDF_SALT, CSRF_CONTEXT, TOKEN_BYTES))
|
||||
.toString("base64url");
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
export function createFileAuthSessionStore(
|
||||
root: string,
|
||||
validity?: AuthSessionValidity,
|
||||
options: FileAuthSessionStoreOptions = {},
|
||||
): AuthSessionStore {
|
||||
const oidcStateCapacity = options.oidcStateCapacity ?? OIDC_STATE_CAPACITY;
|
||||
if (!Number.isInteger(oidcStateCapacity) || oidcStateCapacity < 1 || oidcStateCapacity > OIDC_STATE_CAPACITY) {
|
||||
throw invalid();
|
||||
}
|
||||
const rawStorage = process.platform === "win32"
|
||||
? options.windowsStorageBridge ?? createWindowsAuthStorageBridge()
|
||||
: options.posixStorageBridge ?? createPosixAuthStorageBridge();
|
||||
// A malformed or failed helper must be indistinguishable from any other storage failure to
|
||||
// callers. This also keeps narrow test seams from accidentally exposing transport details.
|
||||
const storage: WindowsAuthStorageBridge = {
|
||||
validateRoot: async (value) => { try { await rawStorage.validateRoot(value); } catch { throw invalid(); } },
|
||||
ensureLayout: async (value) => { try { await rawStorage.ensureLayout(value); } catch { throw invalid(); } },
|
||||
readAuthConfig: (value) => { try { return rawStorage.readAuthConfig(value); } catch { throw invalid(); } },
|
||||
readLocalUsers: async (value) => { try { return await rawStorage.readLocalUsers(value); } catch { throw invalid(); } },
|
||||
create: async (...args) => { try { return await rawStorage.create(...args); } catch { throw invalid(); } },
|
||||
read: async (...args) => { try { return await rawStorage.read(...args); } catch { throw invalid(); } },
|
||||
replace: async (...args) => { try { await rawStorage.replace(...args); } catch { throw invalid(); } },
|
||||
remove: async (...args) => { try { return await rawStorage.remove(...args); } catch { throw invalid(); } },
|
||||
list: async (...args) => { try { return await rawStorage.list(...args); } catch { throw invalid(); } },
|
||||
listPage: async (...args) => { try { return await rawStorage.listPage(...args); } catch { throw invalid(); } },
|
||||
claimConsume: async (...args) => { try { return await rawStorage.claimConsume(...args); } catch { throw invalid(); } },
|
||||
readClaim: async (...args) => { try { return await rawStorage.readClaim(...args); } catch { throw invalid(); } },
|
||||
removeClaim: async (...args) => { try { return await rawStorage.removeClaim(...args); } catch { throw invalid(); } },
|
||||
};
|
||||
let sessionPruneCursor: string | undefined;
|
||||
|
||||
function requiredStorage(): WindowsAuthStorageBridge {
|
||||
if (!storage) throw invalid();
|
||||
return storage;
|
||||
}
|
||||
|
||||
async function ordinarySessionPage(after: string | undefined): Promise<SessionDirectoryPage> {
|
||||
const page = await requiredStorage().listPage(root, "sessions", after, MAX_SESSION_PRUNE_ENTRIES);
|
||||
if (!page || !Array.isArray(page.entries) || typeof page.more !== "boolean") throw invalid();
|
||||
return { entries: page.entries.map((entry) => entry.name), more: page.more };
|
||||
}
|
||||
|
||||
function nextSessionPruneCursor(page: SessionDirectoryPage, after: string | undefined): string | undefined {
|
||||
if (!Array.isArray(page.entries) || typeof page.more !== "boolean"
|
||||
|| page.entries.length > MAX_SESSION_PRUNE_ENTRIES) throw invalid();
|
||||
const seen = new Set<string>();
|
||||
let previous = after;
|
||||
for (const filename of page.entries) {
|
||||
if (!DIGEST_FILENAME_PATTERN.test(filename) || seen.has(filename)
|
||||
|| (previous !== undefined && filename <= previous)) throw invalid();
|
||||
seen.add(filename);
|
||||
previous = filename;
|
||||
}
|
||||
if (!page.more) return undefined;
|
||||
if (page.entries.length !== MAX_SESSION_PRUNE_ENTRIES || previous === undefined || previous === after) throw invalid();
|
||||
return previous;
|
||||
}
|
||||
|
||||
async function pruneOrdinarySessions(nowMs: number): Promise<number> {
|
||||
const after = sessionPruneCursor;
|
||||
const page = await ordinarySessionPage(after);
|
||||
const next = nextSessionPruneCursor(page, after);
|
||||
let removed = 0;
|
||||
const bridge = requiredStorage();
|
||||
for (const filename of page.entries) {
|
||||
const contents = await bridge.read(root, "sessions", filename);
|
||||
if (!contents) continue;
|
||||
const record = parseWindowsRecord(contents, MAX_SESSION_RECORD_BYTES, parseSessionRecord);
|
||||
if (sessionExpired(record, nowMs) && await bridge.remove(root, "sessions", filename)) removed += 1;
|
||||
}
|
||||
sessionPruneCursor = next;
|
||||
return removed;
|
||||
}
|
||||
|
||||
async function oidcStorageEntries(): Promise<string[]> {
|
||||
const entries = (await requiredStorage().list(root, "oidc", MAX_OIDC_STORAGE_ENTRIES)).map((entry) => entry.name);
|
||||
if (entries.length > MAX_OIDC_STORAGE_ENTRIES) throw invalid();
|
||||
if (entries.some((entry) => !DIGEST_FILENAME_PATTERN.test(entry)
|
||||
&& !CLAIM_FILENAME_PATTERN.test(entry) && oidcSlotIndex(entry) === undefined)) throw invalid();
|
||||
return entries;
|
||||
}
|
||||
|
||||
async function storedOidcSlots(suppliedEntries?: string[]): Promise<StoredOidcSlot[]> {
|
||||
const entries = suppliedEntries ?? await oidcStorageEntries();
|
||||
const slots: StoredOidcSlot[] = [];
|
||||
const stateFilenames = new Set<string>();
|
||||
for (const filename of entries) {
|
||||
const index = oidcSlotIndex(filename);
|
||||
if (index === undefined) continue;
|
||||
const contents = await requiredStorage().read(root, "oidc", filename);
|
||||
if (!contents) continue;
|
||||
const record = parseWindowsRecord(contents, MAX_OIDC_SLOT_RECORD_BYTES, parseOidcSlotRecord);
|
||||
if (stateFilenames.has(record.stateFilename)) throw invalid();
|
||||
stateFilenames.add(record.stateFilename);
|
||||
slots.push({ filename, index, record });
|
||||
}
|
||||
return slots;
|
||||
}
|
||||
|
||||
async function removeOidcSlot(slot: StoredOidcSlot): Promise<void> {
|
||||
const current = await requiredStorage().read(root, "oidc", slot.filename);
|
||||
if (!current) return;
|
||||
const record = parseWindowsRecord(current, MAX_OIDC_SLOT_RECORD_BYTES, parseOidcSlotRecord);
|
||||
if (record.stateFilename !== slot.record.stateFilename || record.expiresAt !== slot.record.expiresAt
|
||||
|| !await requiredStorage().remove(root, "oidc", slot.filename)) throw invalid();
|
||||
}
|
||||
|
||||
async function releaseOidcSlot(index: number | undefined, stateFilename: string): Promise<void> {
|
||||
if (index === undefined) return;
|
||||
const filename = oidcSlotFilename(index);
|
||||
const slot = (await storedOidcSlots([filename]))[0];
|
||||
if (!slot || slot.record.stateFilename !== stateFilename) throw invalid();
|
||||
await removeOidcSlot(slot);
|
||||
}
|
||||
|
||||
async function reserveOidcSlot(stateFilename: string, expiresAt: string): Promise<number> {
|
||||
const entries = await oidcStorageEntries();
|
||||
const slots = await storedOidcSlots(entries);
|
||||
const representedStates = new Set(slots.map((slot) => slot.record.stateFilename));
|
||||
const legacyStates = new Set<string>();
|
||||
for (const entry of entries) {
|
||||
const filename = CLAIM_FILENAME_PATTERN.test(entry)
|
||||
? `${entry.slice(0, -".claim".length)}.json`
|
||||
: entry;
|
||||
if (DIGEST_FILENAME_PATTERN.test(filename) && !representedStates.has(filename)) legacyStates.add(filename);
|
||||
}
|
||||
const availableSlotCount = oidcStateCapacity - legacyStates.size;
|
||||
if (availableSlotCount <= 0) throw new OidcStateCapacityError();
|
||||
const occupied = new Set(slots.map((slot) => slot.index));
|
||||
const record: OidcSlotRecord = { version: 1, stateFilename, expiresAt };
|
||||
const contents = serialize(record, MAX_OIDC_SLOT_RECORD_BYTES);
|
||||
for (let index = 0; index < availableSlotCount; index += 1) {
|
||||
if (occupied.has(index)) continue;
|
||||
const filename = oidcSlotFilename(index);
|
||||
const created = await requiredStorage().create(root, "oidc", filename, contents);
|
||||
if (created) return index;
|
||||
}
|
||||
throw new OidcStateCapacityError();
|
||||
}
|
||||
|
||||
async function createSession(input: SessionCreateInput, now = new Date()): Promise<CreatedAuthSession> {
|
||||
const nowMs = dateMilliseconds(now);
|
||||
let validated: z.infer<typeof sessionInputSchema>;
|
||||
try {
|
||||
validated = sessionInputSchema.parse(input);
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
const absoluteExpiresMs = nowMs + validated.absoluteTtlMs;
|
||||
const idleExpiresMs = Math.min(nowMs + validated.idleTtlMs, absoluteExpiresMs);
|
||||
if (!Number.isSafeInteger(absoluteExpiresMs) || !Number.isSafeInteger(idleExpiresMs)) throw invalid();
|
||||
const record: AuthSessionRecord = {
|
||||
version: 1,
|
||||
issuer: validated.principal.issuer,
|
||||
subject: validated.principal.subject,
|
||||
...(validated.principal.displayName === undefined ? {} : { displayName: validated.principal.displayName }),
|
||||
method: validated.method,
|
||||
roles: [...validated.principal.roles],
|
||||
permissions: [...validated.principal.permissions],
|
||||
...(validated.userAuthRevision === undefined ? {} : { userAuthRevision: validated.userAuthRevision }),
|
||||
authConfigRevision: validated.authConfigRevision,
|
||||
remembered: validated.remembered,
|
||||
createdAt: isoAt(nowMs),
|
||||
lastSeenAt: isoAt(nowMs),
|
||||
idleExpiresAt: isoAt(idleExpiresMs),
|
||||
absoluteExpiresAt: isoAt(absoluteExpiresMs),
|
||||
};
|
||||
const contents = serialize(record, MAX_SESSION_RECORD_BYTES);
|
||||
const bridge = requiredStorage();
|
||||
for (let attempt = 0; attempt < 8; attempt += 1) {
|
||||
const token = randomBytes(TOKEN_BYTES).toString("base64url");
|
||||
const filename = digestFilename(token);
|
||||
if (await bridge.create(root, "sessions", filename, contents)) {
|
||||
return { token, csrfToken: deriveCsrfToken(token), record };
|
||||
}
|
||||
}
|
||||
throw invalid();
|
||||
}
|
||||
|
||||
async function resolveSession(
|
||||
token: string,
|
||||
now = new Date(),
|
||||
requestValidity = validity,
|
||||
): Promise<AuthSessionRecord | undefined> {
|
||||
if (!canonicalRawValue(token)) return undefined;
|
||||
const nowMs = dateMilliseconds(now);
|
||||
const filename = digestFilename(token);
|
||||
return withLock(lockKey(root, "sessions", filename), async () => {
|
||||
const bridge = requiredStorage();
|
||||
const contents = await bridge.read(root, "sessions", filename);
|
||||
if (!contents) return undefined;
|
||||
const record = parseWindowsRecord(contents, MAX_SESSION_RECORD_BYTES, parseSessionRecord);
|
||||
if (sessionExpired(record, nowMs)) {
|
||||
await bridge.remove(root, "sessions", filename);
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
if (await recordIsCurrent(record, requestValidity)) return record;
|
||||
} catch (error) {
|
||||
if (error instanceof AuthSessionOperationalError) throw error;
|
||||
await bridge.remove(root, "sessions", filename);
|
||||
throw invalid();
|
||||
}
|
||||
await bridge.remove(root, "sessions", filename);
|
||||
return undefined;
|
||||
});
|
||||
}
|
||||
|
||||
async function touchSession(token: string, now = new Date()): Promise<void> {
|
||||
if (!canonicalRawValue(token)) return;
|
||||
const nowMs = dateMilliseconds(now);
|
||||
const filename = digestFilename(token);
|
||||
await withLock(lockKey(root, "sessions", filename), async () => {
|
||||
const bridge = requiredStorage();
|
||||
const contents = await bridge.read(root, "sessions", filename);
|
||||
if (!contents) return;
|
||||
const record = parseWindowsRecord(contents, MAX_SESSION_RECORD_BYTES, parseSessionRecord);
|
||||
if (sessionExpired(record, nowMs)) {
|
||||
await bridge.remove(root, "sessions", filename);
|
||||
return undefined;
|
||||
}
|
||||
const lastSeenMs = Date.parse(record.lastSeenAt);
|
||||
if (nowMs <= lastSeenMs || nowMs - lastSeenMs < TOUCH_INTERVAL_MS) return;
|
||||
const idleWindowMs = Date.parse(record.idleExpiresAt) - lastSeenMs;
|
||||
if (idleWindowMs <= 0 || idleWindowMs > MAX_TTL_MS) throw invalid();
|
||||
const touched: AuthSessionRecord = {
|
||||
...record,
|
||||
lastSeenAt: isoAt(nowMs),
|
||||
idleExpiresAt: isoAt(Math.min(nowMs + idleWindowMs, Date.parse(record.absoluteExpiresAt))),
|
||||
};
|
||||
await bridge.replace(root, "sessions", filename, serialize(touched, MAX_SESSION_RECORD_BYTES));
|
||||
});
|
||||
}
|
||||
|
||||
async function revokeSession(token: string): Promise<void> {
|
||||
if (!canonicalRawValue(token)) return;
|
||||
const filename = digestFilename(token);
|
||||
await withLock(lockKey(root, "sessions", filename), async () => {
|
||||
await requiredStorage().remove(root, "sessions", filename);
|
||||
});
|
||||
}
|
||||
|
||||
async function pruneOidcStates(nowMs: number): Promise<number> {
|
||||
const bridge = requiredStorage();
|
||||
const oidcEntries = await bridge.list(root, "oidc", MAX_OIDC_STORAGE_ENTRIES);
|
||||
if (oidcEntries.length > MAX_OIDC_STORAGE_ENTRIES) throw invalid();
|
||||
const stateNames = new Set(oidcEntries
|
||||
.filter((entry) => DIGEST_FILENAME_PATTERN.test(entry.name))
|
||||
.map((entry) => entry.name));
|
||||
const claimEntries = new Map(oidcEntries
|
||||
.filter((entry) => CLAIM_FILENAME_PATTERN.test(entry.name))
|
||||
.map((entry) => [entry.name, entry]));
|
||||
const slotEntries = oidcEntries.filter((entry) => oidcSlotIndex(entry.name) !== undefined);
|
||||
if (stateNames.size + claimEntries.size + slotEntries.length !== oidcEntries.length) throw invalid();
|
||||
let removed = 0;
|
||||
for (const filename of stateNames) {
|
||||
const claim = claimFilename(filename);
|
||||
const contents = claimEntries.has(claim)
|
||||
? await bridge.readClaim(root, filename)
|
||||
: await bridge.read(root, "oidc", filename);
|
||||
if (!contents) continue;
|
||||
const record = parseWindowsRecord(contents, MAX_OIDC_STATE_RECORD_BYTES, parseOidcStateRecord);
|
||||
if (oidcStateExpired(record, nowMs)) {
|
||||
const didRemove = claimEntries.has(claim)
|
||||
? await bridge.removeClaim(root, filename)
|
||||
: await bridge.remove(root, "oidc", filename);
|
||||
if (didRemove) removed += 1;
|
||||
}
|
||||
}
|
||||
for (const [claim, entry] of claimEntries) {
|
||||
const filename = `${claim.slice(0, -".claim".length)}.json`;
|
||||
if (stateNames.has(filename)) continue;
|
||||
if (nowMs >= entry.modifiedUnixMs + OIDC_STATE_TTL_MS
|
||||
&& await bridge.remove(root, "oidc", claim)) removed += 1;
|
||||
}
|
||||
for (const entry of slotEntries) {
|
||||
const contents = await bridge.read(root, "oidc", entry.name);
|
||||
if (!contents) continue;
|
||||
const slot = parseWindowsRecord(contents, MAX_OIDC_SLOT_RECORD_BYTES, parseOidcSlotRecord);
|
||||
if (nowMs < Date.parse(slot.expiresAt)) continue;
|
||||
const claim = claimFilename(slot.stateFilename);
|
||||
const stateContents = claimEntries.has(claim)
|
||||
? await bridge.readClaim(root, slot.stateFilename)
|
||||
: await bridge.read(root, "oidc", slot.stateFilename);
|
||||
if (stateContents) {
|
||||
const state = parseWindowsRecord(stateContents, MAX_OIDC_STATE_RECORD_BYTES, parseOidcStateRecord);
|
||||
if (!oidcStateExpired(state, nowMs)) throw invalid();
|
||||
const didRemove = claimEntries.has(claim)
|
||||
? await bridge.removeClaim(root, slot.stateFilename)
|
||||
: await bridge.remove(root, "oidc", slot.stateFilename);
|
||||
if (didRemove) removed += 1;
|
||||
}
|
||||
if (!await bridge.remove(root, "oidc", entry.name)) throw invalid();
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
async function createOidcState(input: OidcStateCreateInput, now = new Date()): Promise<CreatedOidcState> {
|
||||
const nowMs = dateMilliseconds(now);
|
||||
let validated: z.infer<typeof oidcStateInputSchema>;
|
||||
try {
|
||||
validated = oidcStateInputSchema.parse(input);
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
const expiresMs = nowMs + OIDC_STATE_TTL_MS;
|
||||
if (!Number.isSafeInteger(expiresMs)) throw invalid();
|
||||
return withLock(lockKey(root, "oidc", "capacity"), async () => {
|
||||
await pruneOidcStates(nowMs);
|
||||
for (let attempt = 0; attempt < 8; attempt += 1) {
|
||||
const state = randomBytes(TOKEN_BYTES).toString("base64url");
|
||||
const filename = digestFilename(state);
|
||||
const capacitySlot = await reserveOidcSlot(filename, isoAt(expiresMs));
|
||||
const record: OidcStateRecord = {
|
||||
version: 1,
|
||||
nonce: validated.nonce,
|
||||
codeVerifier: validated.codeVerifier,
|
||||
returnTo: validated.returnTo,
|
||||
authConfigRevision: validated.authConfigRevision,
|
||||
issuer: validated.issuer,
|
||||
browserTransactionDigest: validated.browserTransactionDigest,
|
||||
browserTransactionTransport: validated.browserTransactionTransport,
|
||||
capacitySlot,
|
||||
createdAt: isoAt(nowMs),
|
||||
expiresAt: isoAt(expiresMs),
|
||||
};
|
||||
const contents = serialize(record, MAX_OIDC_STATE_RECORD_BYTES);
|
||||
const created = await requiredStorage().create(root, "oidc", filename, contents);
|
||||
if (created) return { state, record };
|
||||
await releaseOidcSlot(capacitySlot, filename);
|
||||
}
|
||||
throw invalid();
|
||||
});
|
||||
}
|
||||
|
||||
async function consumeOidcState(state: string, now = new Date()): Promise<OidcStateRecord | undefined> {
|
||||
if (!canonicalRawValue(state)) return undefined;
|
||||
const nowMs = dateMilliseconds(now);
|
||||
const filename = digestFilename(state);
|
||||
return withLock(lockKey(root, "oidc", filename), async () => {
|
||||
const contents = await requiredStorage().claimConsume(root, filename);
|
||||
if (!contents) return undefined;
|
||||
const record = parseWindowsRecord(contents, MAX_OIDC_STATE_RECORD_BYTES, parseOidcStateRecord);
|
||||
await releaseOidcSlot(record.capacitySlot, filename);
|
||||
return oidcStateExpired(record, nowMs) ? undefined : record;
|
||||
});
|
||||
}
|
||||
|
||||
async function prune(now = new Date()): Promise<number> {
|
||||
const nowMs = dateMilliseconds(now);
|
||||
// Cursor advancement is process-local, so concurrent timer/manual invocations must not
|
||||
// observe the same page and strand a later page forever.
|
||||
return await withLock(lockKey(root, "sessions", "maintenance"), async () =>
|
||||
(await pruneOrdinarySessions(nowMs)) + (await pruneOidcStates(nowMs)));
|
||||
}
|
||||
|
||||
return {
|
||||
create: createSession,
|
||||
resolve: resolveSession,
|
||||
touch: touchSession,
|
||||
revoke: revokeSession,
|
||||
prune,
|
||||
createOidcState,
|
||||
consumeOidcState,
|
||||
};
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
export type AuthMode = "local" | "oidc" | "upstream" | "none" | "mock";
|
||||
|
||||
export type Role = "user" | "admin";
|
||||
|
||||
export type Permission =
|
||||
| "session.use" | "session.read_all" | "session.manage_all"
|
||||
| "settings.manage" | "workspace.manage" | "workspace.secrets.manage"
|
||||
| "database.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
|
||||
export interface AuthenticationSessionConfig {
|
||||
regularTtlSeconds: number;
|
||||
regularIdleSeconds: number;
|
||||
rememberTtlSeconds: number;
|
||||
rememberIdleSeconds: number;
|
||||
oidcTtlSeconds: number;
|
||||
}
|
||||
|
||||
export interface LocalAuthenticationConfig {
|
||||
version: 1;
|
||||
mode: "local";
|
||||
publicUrl: string;
|
||||
session: AuthenticationSessionConfig;
|
||||
local: { usersFile: string };
|
||||
}
|
||||
|
||||
export interface OidcAuthenticationConfig {
|
||||
version: 1;
|
||||
mode: "oidc";
|
||||
publicUrl: string;
|
||||
session: AuthenticationSessionConfig;
|
||||
oidc: {
|
||||
issuer: string;
|
||||
clientId: string;
|
||||
clientSecretRef: "THT_OIDC_CLIENT_SECRET";
|
||||
scopes: readonly string[];
|
||||
groupsClaim: "groups";
|
||||
};
|
||||
groupCatalog: {
|
||||
driver: "authentik";
|
||||
baseUrl: string;
|
||||
apiTokenRef: "THT_AUTHENTIK_API_TOKEN";
|
||||
};
|
||||
authorization: { groupRoles: Readonly<Record<string, readonly Role[]>> };
|
||||
}
|
||||
|
||||
export type AuthenticationConfig = LocalAuthenticationConfig | OidcAuthenticationConfig;
|
||||
|
||||
export interface LocalUserRecord {
|
||||
id: string;
|
||||
username: string;
|
||||
normalizedUsername: string;
|
||||
displayName?: string;
|
||||
passwordHash: string;
|
||||
roles: readonly Role[];
|
||||
enabled: boolean;
|
||||
authRevision: number;
|
||||
}
|
||||
|
||||
export interface RuntimeProjectionSnapshot {
|
||||
generation: string;
|
||||
canonicalRevision: string;
|
||||
localUsers?: readonly LocalUserRecord[];
|
||||
}
|
||||
|
||||
export interface LoadedAuthConfig {
|
||||
value: AuthenticationConfig;
|
||||
revision: string;
|
||||
sourcePath: string;
|
||||
runtimeProjection?: RuntimeProjectionSnapshot;
|
||||
}
|
||||
|
||||
export interface AuthenticationConfigProvider {
|
||||
current(): LoadedAuthConfig;
|
||||
}
|
||||
|
||||
/** The only browser transport modes accepted for an OIDC transaction cookie. */
|
||||
export type OidcTransactionTransport = "https" | "loopback_http";
|
||||
|
||||
/** Durable, server-side representation of an opaque browser session. */
|
||||
export interface AuthSessionRecord {
|
||||
version: 1;
|
||||
issuer: string;
|
||||
subject: string;
|
||||
displayName?: string;
|
||||
method: "local" | "oidc" | "upstream";
|
||||
roles: readonly Role[];
|
||||
permissions: readonly Permission[];
|
||||
userAuthRevision?: number;
|
||||
authConfigRevision: string;
|
||||
remembered: boolean;
|
||||
createdAt: string;
|
||||
lastSeenAt: string;
|
||||
idleExpiresAt: string;
|
||||
absoluteExpiresAt: string;
|
||||
}
|
||||
|
||||
/** Server-side OIDC callback material keyed by a separately generated opaque state value. */
|
||||
export interface OidcStateRecord {
|
||||
version: 1;
|
||||
nonce: string;
|
||||
codeVerifier: string;
|
||||
returnTo: "/";
|
||||
authConfigRevision: string;
|
||||
issuer: string;
|
||||
browserTransactionDigest: string;
|
||||
/** Optional only to safely consume and reject a short-lived pre-transport legacy state. */
|
||||
browserTransactionTransport?: OidcTransactionTransport;
|
||||
capacitySlot?: number;
|
||||
createdAt: string;
|
||||
expiresAt: string;
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
export interface ConfiguredTransportUrlOptions {
|
||||
allowLoopbackHttp: boolean;
|
||||
originOnly?: boolean;
|
||||
}
|
||||
|
||||
export function parseCredentialFreeHttpUrl(value: string): URL | undefined {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(value);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (!["http:", "https:"].includes(url.protocol)
|
||||
|| url.username || url.password || url.search || url.hash) return undefined;
|
||||
return url;
|
||||
}
|
||||
|
||||
function canonicalLoopbackAuthority(value: string): boolean {
|
||||
const match = /^http:\/\/([^/?#]+)(?:[/?#]|$)/.exec(value);
|
||||
if (!match) return false;
|
||||
const authority = match[1];
|
||||
let port: string | undefined;
|
||||
if (authority.startsWith("[")) {
|
||||
const ipv6 = /^(\[::1\])(?::([^:]+))?$/.exec(authority);
|
||||
if (!ipv6) return false;
|
||||
port = ipv6[2];
|
||||
} else {
|
||||
const ipv4 = /^([^:]+)(?::([^:]+))?$/.exec(authority);
|
||||
if (!ipv4) return false;
|
||||
const octets = ipv4[1].split(".");
|
||||
if (octets.length !== 4 || octets.some((octet) => !/^(?:0|[1-9]\d{0,2})$/.test(octet)
|
||||
|| Number(octet) > 255) || Number(octets[0]) !== 127) return false;
|
||||
port = ipv4[2];
|
||||
}
|
||||
return port === undefined || (/^(?:0|[1-9]\d{0,4})$/.test(port) && Number(port) <= 65_535);
|
||||
}
|
||||
|
||||
export function parseConfiguredTransportUrl(
|
||||
value: string,
|
||||
options: ConfiguredTransportUrlOptions,
|
||||
): URL | undefined {
|
||||
const url = parseCredentialFreeHttpUrl(value);
|
||||
if (!url || (options.originOnly && url.pathname !== "/")) return undefined;
|
||||
if (url.protocol === "https:") return url;
|
||||
if (options.allowLoopbackHttp && url.protocol === "http:" && canonicalLoopbackAuthority(value)) return url;
|
||||
return undefined;
|
||||
}
|
||||
@@ -1,639 +0,0 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { posix, win32 } from "node:path";
|
||||
import type { Readable, Writable } from "node:stream";
|
||||
import { z } from "zod";
|
||||
|
||||
const PROTOCOL_VERSION = 1;
|
||||
const MAX_PROTOCOL_BYTES = 64 * 1024;
|
||||
const MAX_RESPONSE_BYTES = 64 * 1024;
|
||||
const MAX_AUTH_CONFIG_BYTES = 1024 * 1024;
|
||||
const MAX_AUTH_CONFIG_BASE64_BYTES = 4 * Math.ceil(MAX_AUTH_CONFIG_BYTES / 3);
|
||||
const MAX_AUTH_CONFIG_RESPONSE_BYTES = MAX_AUTH_CONFIG_BASE64_BYTES + 1024;
|
||||
const MAX_SESSION_BYTES = 16 * 1024;
|
||||
const MAX_OIDC_BYTES = 8 * 1024;
|
||||
const DEFAULT_MAX_ENTRIES = 256;
|
||||
const MAX_ENTRIES = 512;
|
||||
const TIMEOUT_MS = 5_000;
|
||||
const TERMINATION_GRACE_MS = 100;
|
||||
const FINAL_SETTLEMENT_MS = 750;
|
||||
const DIGEST_FILENAME = /^[a-f0-9]{64}\.json$/;
|
||||
const CLAIM_FILENAME = /^[a-f0-9]{64}\.claim$/;
|
||||
const OIDC_SLOT_FILENAME = /^slot-(?:[0-5][0-9]|6[0-3])\.json$/;
|
||||
const AUTH_CONFIG_FILENAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,249}\.yaml$/;
|
||||
|
||||
const invalid = (): Error => new Error("auth_session_store_invalid");
|
||||
|
||||
export type WindowsAuthStorageDirectory = "sessions" | "oidc";
|
||||
|
||||
export interface WindowsAuthStorageEntry {
|
||||
name: string;
|
||||
modifiedUnixMs: number;
|
||||
}
|
||||
|
||||
export interface WindowsAuthStoragePage {
|
||||
entries: WindowsAuthStorageEntry[];
|
||||
more: boolean;
|
||||
}
|
||||
|
||||
/** Internal adapter boundary for the file-session store's native Windows path. */
|
||||
export interface WindowsAuthStorageBridge {
|
||||
validateRoot(root: string): Promise<void>;
|
||||
ensureLayout(root: string): Promise<void>;
|
||||
readAuthConfig(path: string): Buffer;
|
||||
readLocalUsers(path: string): Promise<Buffer>;
|
||||
create(root: string, directory: WindowsAuthStorageDirectory, filename: string, contents: Buffer): Promise<boolean>;
|
||||
read(root: string, directory: WindowsAuthStorageDirectory, filename: string): Promise<Buffer | undefined>;
|
||||
replace(root: string, directory: WindowsAuthStorageDirectory, filename: string, contents: Buffer): Promise<void>;
|
||||
remove(root: string, directory: WindowsAuthStorageDirectory, filename: string): Promise<boolean>;
|
||||
list(
|
||||
root: string,
|
||||
directory: WindowsAuthStorageDirectory,
|
||||
maximumEntries?: number,
|
||||
): Promise<WindowsAuthStorageEntry[]>;
|
||||
listPage(
|
||||
root: string,
|
||||
directory: "sessions",
|
||||
afterName: string | undefined,
|
||||
maximumEntries: number,
|
||||
): Promise<WindowsAuthStoragePage>;
|
||||
claimConsume(root: string, filename: string): Promise<Buffer | undefined>;
|
||||
readClaim(root: string, filename: string): Promise<Buffer | undefined>;
|
||||
removeClaim(root: string, filename: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface WindowsAuthStorageInvocation {
|
||||
executable: string;
|
||||
args: readonly string[];
|
||||
input: Buffer;
|
||||
timeoutMs: number;
|
||||
maximumOutputBytes: number;
|
||||
}
|
||||
|
||||
export interface WindowsAuthStorageInvocationResult {
|
||||
code: number;
|
||||
stdout: Buffer;
|
||||
stderr: Buffer;
|
||||
}
|
||||
|
||||
interface WindowsAuthStorageChild {
|
||||
readonly stdin: Writable | null;
|
||||
readonly stdout: Readable | null;
|
||||
readonly stderr: Readable | null;
|
||||
kill(signal?: NodeJS.Signals | number): boolean;
|
||||
unref?(): void;
|
||||
on(event: "error", listener: (error: Error) => void): this;
|
||||
once(event: "error", listener: (error: Error) => void): this;
|
||||
once(event: "close", listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
|
||||
removeListener?(event: "error" | "close", listener: (...args: any[]) => void): this;
|
||||
}
|
||||
|
||||
type WindowsAuthStorageSpawn = (
|
||||
executable: string,
|
||||
args: readonly string[],
|
||||
options: { shell: false; windowsHide: true; stdio: ["pipe", "pipe", "pipe"]; env: NodeJS.ProcessEnv },
|
||||
) => WindowsAuthStorageChild;
|
||||
|
||||
export interface WindowsAuthStorageBridgeOptions {
|
||||
/** Test-only transport seam. Production always uses the no-shell child-process invocation. */
|
||||
invoke?: (invocation: WindowsAuthStorageInvocation) => Promise<WindowsAuthStorageInvocationResult>;
|
||||
/** Test-only synchronous seam used by the synchronous authentication-config provider. */
|
||||
invokeSync?: (invocation: WindowsAuthStorageInvocation) => WindowsAuthStorageInvocationResult;
|
||||
/** Optional configured tht path. Defaults to THT_BIN, then the safe bare command `tht`. */
|
||||
thtExecutable?: string;
|
||||
/** Test-only child-launch seam; production uses the fixed no-shell Node child-process launcher. */
|
||||
spawnChild?: WindowsAuthStorageSpawn;
|
||||
/** Test-only input scheduling seam for real child-process lifecycle tests. */
|
||||
beforeInputForTest?: () => Promise<void>;
|
||||
/** Test-only bounded lifecycle timings. Production always uses the fixed deadlines below. */
|
||||
deadlinesForTest?: {
|
||||
timeoutMs?: number;
|
||||
terminationGraceMs?: number;
|
||||
finalSettlementMs?: number;
|
||||
};
|
||||
}
|
||||
|
||||
type AuthStoragePathStyle = "posix" | "windows";
|
||||
|
||||
const responseSchema = z.strictObject({
|
||||
version: z.literal(PROTOCOL_VERSION),
|
||||
ok: z.literal(true),
|
||||
created: z.boolean().optional(),
|
||||
replaced: z.boolean().optional(),
|
||||
removed: z.boolean().optional(),
|
||||
found: z.boolean().optional(),
|
||||
contentBase64: z.string().max(MAX_AUTH_CONFIG_BASE64_BYTES).optional(),
|
||||
entries: z.array(z.strictObject({
|
||||
name: z.string().max(128),
|
||||
modifiedUnixMs: z.number().int().safe().nonnegative(),
|
||||
})).max(MAX_ENTRIES).optional(),
|
||||
more: z.boolean().optional(),
|
||||
validated: z.boolean().optional(),
|
||||
prepared: z.boolean().optional(),
|
||||
});
|
||||
|
||||
type BridgeResponse = z.infer<typeof responseSchema>;
|
||||
|
||||
interface BridgeRequest {
|
||||
version: typeof PROTOCOL_VERSION;
|
||||
operation: "validate-root" | "ensure-layout" | "read-auth-config" | "read-local-users" | "create" | "read" | "replace" | "remove" | "list" | "claim-consume" | "read-claim" | "remove-claim";
|
||||
root: string;
|
||||
directory?: WindowsAuthStorageDirectory;
|
||||
filename?: string;
|
||||
contentBase64?: string;
|
||||
maximumEntries?: number;
|
||||
afterName?: string;
|
||||
continuation?: true;
|
||||
}
|
||||
|
||||
function directoryMaximum(directory: WindowsAuthStorageDirectory): number {
|
||||
return directory === "sessions" ? MAX_SESSION_BYTES : MAX_OIDC_BYTES;
|
||||
}
|
||||
|
||||
function canonicalBase64(value: string, maximum: number): Buffer {
|
||||
if (typeof value !== "string" || value.length > Math.ceil(maximum / 3) * 4) throw invalid();
|
||||
try {
|
||||
const decoded = Buffer.from(value, "base64");
|
||||
if (decoded.length === 0 || decoded.length > maximum || decoded.toString("base64") !== value) throw invalid();
|
||||
return decoded;
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function validateRoot(root: string, pathStyle: AuthStoragePathStyle): void {
|
||||
const paths = pathStyle === "windows" ? win32 : posix;
|
||||
if (typeof root !== "string" || root.length === 0 || /[\u0000-\u001f\u007f]/.test(root)
|
||||
|| !paths.isAbsolute(root) || paths.normalize(root) !== root) throw invalid();
|
||||
}
|
||||
|
||||
function validateFilename(filename: string, allowClaim = false, allowOidcSlot = false): void {
|
||||
if (typeof filename !== "string" || (!DIGEST_FILENAME.test(filename)
|
||||
&& !(allowClaim && CLAIM_FILENAME.test(filename))
|
||||
&& !(allowOidcSlot && OIDC_SLOT_FILENAME.test(filename)))) throw invalid();
|
||||
}
|
||||
|
||||
function safeThtExecutable(value: string | undefined, pathStyle: AuthStoragePathStyle): string {
|
||||
const executable = value ?? process.env.THT_AUTH_STORAGE_BIN ?? process.env.THT_BIN ?? "tht";
|
||||
if (typeof executable !== "string" || executable.length === 0 || /[\u0000-\u001f\u007f]/.test(executable)) throw invalid();
|
||||
if (executable === "tht" || (pathStyle === "windows" && executable === "tht.exe")) return executable;
|
||||
const paths = pathStyle === "windows" ? win32 : posix;
|
||||
if (paths.isAbsolute(executable) && paths.normalize(executable) === executable
|
||||
&& (pathStyle === "posix" || /\.exe$/i.test(executable))) return executable;
|
||||
throw invalid();
|
||||
}
|
||||
|
||||
function parseResponse(result: WindowsAuthStorageInvocationResult, maximumOutputBytes: number): BridgeResponse {
|
||||
if (!Number.isInteger(result.code) || result.code !== 0 || !Buffer.isBuffer(result.stdout)
|
||||
|| !Buffer.isBuffer(result.stderr) || result.stderr.length > MAX_RESPONSE_BYTES
|
||||
|| result.stdout.length === 0 || result.stdout.length > maximumOutputBytes) {
|
||||
throw invalid();
|
||||
}
|
||||
try {
|
||||
const source = new TextDecoder("utf-8", { fatal: true }).decode(result.stdout);
|
||||
return responseSchema.parse(JSON.parse(source));
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function encodedRequest(request: BridgeRequest, pathStyle: AuthStoragePathStyle): Buffer {
|
||||
validateRoot(request.root, pathStyle);
|
||||
if (request.operation === "validate-root" || request.operation === "ensure-layout") {
|
||||
if (request.directory !== undefined || request.filename !== undefined || request.contentBase64 !== undefined
|
||||
|| request.maximumEntries !== undefined || request.afterName !== undefined || request.continuation !== undefined) throw invalid();
|
||||
} else if (request.operation === "read-auth-config" || request.operation === "read-local-users") {
|
||||
if (request.directory !== undefined || request.contentBase64 !== undefined || request.maximumEntries !== undefined
|
||||
|| request.afterName !== undefined || request.continuation !== undefined
|
||||
|| request.filename === undefined || !AUTH_CONFIG_FILENAME.test(request.filename)) throw invalid();
|
||||
} else if (request.operation === "list") {
|
||||
if (request.directory === undefined) throw invalid();
|
||||
if (request.filename !== undefined || request.contentBase64 !== undefined) throw invalid();
|
||||
if (request.maximumEntries !== undefined && (!Number.isInteger(request.maximumEntries)
|
||||
|| request.maximumEntries < 1 || request.maximumEntries > MAX_ENTRIES)) throw invalid();
|
||||
if (request.continuation === true) {
|
||||
if (request.directory !== "sessions" || (request.afterName !== undefined && !DIGEST_FILENAME.test(request.afterName))) {
|
||||
throw invalid();
|
||||
}
|
||||
} else if (request.afterName !== undefined || request.continuation !== undefined) {
|
||||
throw invalid();
|
||||
}
|
||||
} else {
|
||||
if (request.directory === undefined) throw invalid();
|
||||
if (request.maximumEntries !== undefined || request.afterName !== undefined || request.continuation !== undefined) throw invalid();
|
||||
if (request.filename === undefined) throw invalid();
|
||||
const allowClaim = request.operation === "remove" && request.directory === "oidc";
|
||||
const allowOidcSlot = request.directory === "oidc"
|
||||
&& (request.operation === "create" || request.operation === "read" || request.operation === "remove");
|
||||
validateFilename(request.filename, allowClaim, allowOidcSlot);
|
||||
if (request.contentBase64 !== undefined && request.operation !== "create" && request.operation !== "replace") throw invalid();
|
||||
}
|
||||
if ((request.operation === "claim-consume" || request.operation === "read-claim" || request.operation === "remove-claim")
|
||||
&& request.directory !== "oidc") throw invalid();
|
||||
if (request.contentBase64 !== undefined) {
|
||||
if (request.directory === undefined) throw invalid();
|
||||
canonicalBase64(request.contentBase64, directoryMaximum(request.directory));
|
||||
}
|
||||
const encoded = Buffer.from(JSON.stringify(request), "utf8");
|
||||
if (encoded.length === 0 || encoded.length > MAX_PROTOCOL_BYTES) throw invalid();
|
||||
return encoded;
|
||||
}
|
||||
|
||||
function environmentForBridge(): NodeJS.ProcessEnv {
|
||||
const path = process.env.PATH;
|
||||
const systemRoot = process.env.SystemRoot ?? process.env.SYSTEMROOT;
|
||||
return {
|
||||
...(path === undefined ? {} : { PATH: path }),
|
||||
...(systemRoot === undefined ? {} : { SystemRoot: systemRoot }),
|
||||
};
|
||||
}
|
||||
|
||||
const spawnTht: WindowsAuthStorageSpawn = (executable, args, options) => spawn(executable, [...args], options);
|
||||
|
||||
function invokeThtSync(invocation: WindowsAuthStorageInvocation): WindowsAuthStorageInvocationResult {
|
||||
try {
|
||||
const result = spawnSync(invocation.executable, [...invocation.args], {
|
||||
shell: false,
|
||||
windowsHide: true,
|
||||
env: environmentForBridge(),
|
||||
input: invocation.input,
|
||||
timeout: invocation.timeoutMs,
|
||||
maxBuffer: invocation.maximumOutputBytes,
|
||||
encoding: "buffer",
|
||||
});
|
||||
if (result.error || result.signal !== null || typeof result.status !== "number"
|
||||
|| !Buffer.isBuffer(result.stdout) || !Buffer.isBuffer(result.stderr)) throw invalid();
|
||||
return { code: result.status, stdout: result.stdout, stderr: result.stderr };
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
async function invokeTht(
|
||||
invocation: WindowsAuthStorageInvocation,
|
||||
spawnChild: WindowsAuthStorageSpawn = spawnTht,
|
||||
beforeInputForTest?: () => Promise<void>,
|
||||
terminationGraceMs = TERMINATION_GRACE_MS,
|
||||
finalSettlementMs = FINAL_SETTLEMENT_MS,
|
||||
): Promise<WindowsAuthStorageInvocationResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let aborted = false;
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
let terminationTimer: NodeJS.Timeout | undefined;
|
||||
let finalSettlementTimer: NodeJS.Timeout | undefined;
|
||||
let lateErrorReleaseTimer: NodeJS.Timeout | undefined;
|
||||
const stdout: Buffer[] = [];
|
||||
const stderr: Buffer[] = [];
|
||||
let stdoutBytes = 0;
|
||||
let stderrBytes = 0;
|
||||
let child: WindowsAuthStorageChild | undefined;
|
||||
let stdin: Writable | undefined;
|
||||
let stdoutStream: Readable | undefined;
|
||||
let stderrStream: Readable | undefined;
|
||||
const swallowChildError = (): void => undefined;
|
||||
const swallowStreamError = (): void => undefined;
|
||||
const releaseQuarantine = (): void => {
|
||||
if (lateErrorReleaseTimer !== undefined) clearTimeout(lateErrorReleaseTimer);
|
||||
lateErrorReleaseTimer = undefined;
|
||||
removeChildListener("error", swallowChildError);
|
||||
removeChildListener("close", releaseQuarantine);
|
||||
removeStreamListener(stdin, "error", swallowStreamError);
|
||||
removeStreamListener(stdoutStream, "error", swallowStreamError);
|
||||
removeStreamListener(stderrStream, "error", swallowStreamError);
|
||||
};
|
||||
const quarantineLateErrors = (): void => {
|
||||
// A final-deadline settlement can precede a broken ChildProcess object's terminal events.
|
||||
// Keep only no-capture listeners for a bounded grace period so a late EventEmitter error
|
||||
// cannot become uncaught, including after an already-observed close event.
|
||||
child?.on("error", swallowChildError);
|
||||
child?.once("close", releaseQuarantine);
|
||||
stdin?.on("error", swallowStreamError);
|
||||
stdoutStream?.on("error", swallowStreamError);
|
||||
stderrStream?.on("error", swallowStreamError);
|
||||
lateErrorReleaseTimer = setTimeout(releaseQuarantine, finalSettlementMs);
|
||||
lateErrorReleaseTimer.unref?.();
|
||||
};
|
||||
const removeChildListener = (event: "error" | "close", listener: (...args: any[]) => void): void => {
|
||||
try { child?.removeListener?.(event, listener); } catch { /* the helper is already terminal */ }
|
||||
};
|
||||
const removeStreamListener = (stream: Writable | Readable | undefined, event: "data" | "error", listener: (...args: any[]) => void): void => {
|
||||
try { stream?.removeListener(event, listener); } catch { /* the helper is already terminal */ }
|
||||
};
|
||||
const stopStream = (stream: Writable | Readable | null | undefined): void => {
|
||||
try { stream?.destroy(); } catch { /* abort is already fail-closed */ }
|
||||
};
|
||||
const onChildError = (): void => abort();
|
||||
const onStdinError = (): void => abort();
|
||||
const onStdoutError = (): void => abort();
|
||||
const onStderrError = (): void => abort();
|
||||
const onStdoutData = (chunk: Buffer): void => {
|
||||
if (aborted || settled) return;
|
||||
stdoutBytes += chunk.length;
|
||||
if (stdoutBytes > invocation.maximumOutputBytes) {
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
stdout.push(Buffer.from(chunk));
|
||||
};
|
||||
const onStderrData = (chunk: Buffer): void => {
|
||||
if (aborted || settled) return;
|
||||
stderrBytes += chunk.length;
|
||||
if (stderrBytes > MAX_RESPONSE_BYTES) {
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
stderr.push(Buffer.from(chunk));
|
||||
};
|
||||
const onClose = (code: number | null, signal: NodeJS.Signals | null): void => {
|
||||
if (settled) return;
|
||||
if (aborted || code === null || !Number.isInteger(code) || signal !== null) {
|
||||
settle(() => reject(invalid()), true);
|
||||
return;
|
||||
}
|
||||
settle(() => resolve({
|
||||
code,
|
||||
stdout: Buffer.concat(stdout),
|
||||
stderr: Buffer.concat(stderr),
|
||||
}));
|
||||
};
|
||||
const cleanup = (quarantine = false): void => {
|
||||
if (timeout !== undefined) clearTimeout(timeout);
|
||||
if (terminationTimer !== undefined) clearTimeout(terminationTimer);
|
||||
if (finalSettlementTimer !== undefined) clearTimeout(finalSettlementTimer);
|
||||
removeChildListener("error", onChildError);
|
||||
removeChildListener("close", onClose as (...args: any[]) => void);
|
||||
removeStreamListener(stdin, "error", onStdinError);
|
||||
removeStreamListener(stdoutStream, "data", onStdoutData);
|
||||
removeStreamListener(stdoutStream, "error", onStdoutError);
|
||||
removeStreamListener(stderrStream, "data", onStderrData);
|
||||
removeStreamListener(stderrStream, "error", onStderrError);
|
||||
if (quarantine) quarantineLateErrors();
|
||||
};
|
||||
const settle = (callback: () => void, quarantine = false): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup(quarantine);
|
||||
callback();
|
||||
};
|
||||
const abort = (): void => {
|
||||
if (aborted || settled) return;
|
||||
aborted = true;
|
||||
if (timeout !== undefined) clearTimeout(timeout);
|
||||
if (child !== undefined) {
|
||||
stopStream(stdin);
|
||||
stopStream(stdoutStream);
|
||||
stopStream(stderrStream);
|
||||
try { child.kill("SIGTERM"); } catch { /* final settlement still owns completion */ }
|
||||
try { child.unref?.(); } catch { /* the bounded timers still own completion */ }
|
||||
}
|
||||
terminationTimer = setTimeout(() => {
|
||||
if (settled || child === undefined) return;
|
||||
try { child.kill("SIGKILL"); } catch { /* final settlement still owns completion */ }
|
||||
}, terminationGraceMs);
|
||||
finalSettlementTimer = setTimeout(() => {
|
||||
settle(() => reject(invalid()), true);
|
||||
}, finalSettlementMs);
|
||||
};
|
||||
try {
|
||||
child = spawnChild(invocation.executable, invocation.args, {
|
||||
shell: false,
|
||||
windowsHide: true,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
env: environmentForBridge(),
|
||||
});
|
||||
} catch {
|
||||
settle(() => reject(invalid()));
|
||||
return;
|
||||
}
|
||||
child.once("close", onClose);
|
||||
child.on("error", onChildError);
|
||||
if (!child.stdin || !child.stdout || !child.stderr) {
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
stdin = child.stdin;
|
||||
stdoutStream = child.stdout;
|
||||
stderrStream = child.stderr;
|
||||
timeout = setTimeout(() => {
|
||||
abort();
|
||||
}, invocation.timeoutMs);
|
||||
stdoutStream.on("data", onStdoutData);
|
||||
stdoutStream.once("error", onStdoutError);
|
||||
stderrStream.on("data", onStderrData);
|
||||
stderrStream.once("error", onStderrError);
|
||||
stdin.once("error", onStdinError);
|
||||
const writeInput = (): void => {
|
||||
if (aborted || settled) return;
|
||||
try {
|
||||
stdin.end(invocation.input);
|
||||
} catch {
|
||||
abort();
|
||||
}
|
||||
};
|
||||
if (beforeInputForTest === undefined) {
|
||||
writeInput();
|
||||
} else {
|
||||
void Promise.resolve().then(beforeInputForTest).then(writeInput, abort);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function contentFrom(response: BridgeResponse, maximum: number): Buffer | undefined {
|
||||
if (response.found !== true) {
|
||||
if (response.contentBase64 !== undefined) throw invalid();
|
||||
return undefined;
|
||||
}
|
||||
if (response.contentBase64 === undefined) throw invalid();
|
||||
return canonicalBase64(response.contentBase64, maximum);
|
||||
}
|
||||
|
||||
function listedEntries(
|
||||
response: BridgeResponse,
|
||||
directory: WindowsAuthStorageDirectory,
|
||||
maximumEntries: number,
|
||||
): WindowsAuthStorageEntry[] {
|
||||
if (response.entries === undefined || response.entries.length > maximumEntries) throw invalid();
|
||||
const names = new Set<string>();
|
||||
for (const entry of response.entries) {
|
||||
if (!DIGEST_FILENAME.test(entry.name) && !(directory === "oidc"
|
||||
&& (CLAIM_FILENAME.test(entry.name) || OIDC_SLOT_FILENAME.test(entry.name)))) throw invalid();
|
||||
if (names.has(entry.name)) throw invalid();
|
||||
names.add(entry.name);
|
||||
}
|
||||
return response.entries.map((entry) => ({ name: entry.name, modifiedUnixMs: entry.modifiedUnixMs }));
|
||||
}
|
||||
|
||||
function createAuthStorageBridge(
|
||||
pathStyle: AuthStoragePathStyle,
|
||||
options: WindowsAuthStorageBridgeOptions = {},
|
||||
): WindowsAuthStorageBridge {
|
||||
const executable = safeThtExecutable(options.thtExecutable, pathStyle);
|
||||
const testDeadlines = options.deadlinesForTest;
|
||||
const timeoutMs = testDeadlines?.timeoutMs ?? TIMEOUT_MS;
|
||||
const terminationGraceMs = testDeadlines?.terminationGraceMs ?? TERMINATION_GRACE_MS;
|
||||
const finalSettlementMs = testDeadlines?.finalSettlementMs ?? FINAL_SETTLEMENT_MS;
|
||||
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > TIMEOUT_MS
|
||||
|| !Number.isSafeInteger(terminationGraceMs) || terminationGraceMs < 1 || terminationGraceMs > TIMEOUT_MS
|
||||
|| !Number.isSafeInteger(finalSettlementMs) || finalSettlementMs <= terminationGraceMs || finalSettlementMs > TIMEOUT_MS) {
|
||||
throw invalid();
|
||||
}
|
||||
const invoke = options.invoke ?? ((invocation: WindowsAuthStorageInvocation) => invokeTht(
|
||||
invocation,
|
||||
options.spawnChild,
|
||||
options.beforeInputForTest,
|
||||
terminationGraceMs,
|
||||
finalSettlementMs,
|
||||
));
|
||||
const invokeSync = options.invokeSync ?? invokeThtSync;
|
||||
const request = async (value: BridgeRequest): Promise<BridgeResponse> => {
|
||||
try {
|
||||
const maximumOutputBytes = value.operation === "read-local-users"
|
||||
? MAX_AUTH_CONFIG_RESPONSE_BYTES
|
||||
: MAX_RESPONSE_BYTES;
|
||||
const response = await invoke({
|
||||
executable,
|
||||
args: ["_auth-storage"],
|
||||
input: encodedRequest(value, pathStyle),
|
||||
timeoutMs,
|
||||
maximumOutputBytes,
|
||||
});
|
||||
return parseResponse(response, maximumOutputBytes);
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
};
|
||||
const syncRequest = (value: BridgeRequest): BridgeResponse => {
|
||||
try {
|
||||
const response = invokeSync({
|
||||
executable,
|
||||
args: ["_auth-storage"],
|
||||
input: encodedRequest(value, pathStyle),
|
||||
timeoutMs,
|
||||
maximumOutputBytes: MAX_AUTH_CONFIG_RESPONSE_BYTES,
|
||||
});
|
||||
return parseResponse(response, MAX_AUTH_CONFIG_RESPONSE_BYTES);
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
};
|
||||
const recordRequest = (operation: "create" | "read" | "replace" | "remove" | "claim-consume" | "read-claim" | "remove-claim", root: string, directory: WindowsAuthStorageDirectory, filename: string, contents?: Buffer): BridgeRequest => ({
|
||||
version: PROTOCOL_VERSION,
|
||||
operation,
|
||||
root,
|
||||
directory,
|
||||
filename,
|
||||
...(contents === undefined ? {} : { contentBase64: contents.toString("base64") }),
|
||||
});
|
||||
|
||||
return {
|
||||
async validateRoot(root) {
|
||||
const response = await request({ version: PROTOCOL_VERSION, operation: "validate-root", root });
|
||||
if (response.validated !== true
|
||||
|| Object.keys(response).some((key) => !["version", "ok", "validated"].includes(key))) throw invalid();
|
||||
},
|
||||
async ensureLayout(root) {
|
||||
const response = await request({ version: PROTOCOL_VERSION, operation: "ensure-layout", root });
|
||||
if (response.prepared !== true
|
||||
|| Object.keys(response).some((key) => !["version", "ok", "prepared"].includes(key))) throw invalid();
|
||||
},
|
||||
readAuthConfig(path) {
|
||||
const paths = pathStyle === "windows" ? win32 : posix;
|
||||
if (typeof path !== "string" || path.length === 0 || /[\u0000-\u001f\u007f]/.test(path)
|
||||
|| !paths.isAbsolute(path) || paths.normalize(path) !== path) throw invalid();
|
||||
const root = paths.dirname(path);
|
||||
const filename = paths.basename(path);
|
||||
if (!AUTH_CONFIG_FILENAME.test(filename) || paths.join(root, filename) !== path) throw invalid();
|
||||
const response = syncRequest({ version: PROTOCOL_VERSION, operation: "read-auth-config", root, filename });
|
||||
if (Object.keys(response).some((key) => !["version", "ok", "found", "contentBase64"].includes(key))) throw invalid();
|
||||
const contents = contentFrom(response, MAX_AUTH_CONFIG_BYTES);
|
||||
if (contents === undefined) throw invalid();
|
||||
return contents;
|
||||
},
|
||||
async readLocalUsers(path) {
|
||||
const paths = pathStyle === "windows" ? win32 : posix;
|
||||
if (typeof path !== "string" || path.length === 0 || /[\u0000-\u001f\u007f]/.test(path)
|
||||
|| !paths.isAbsolute(path) || paths.normalize(path) !== path) throw invalid();
|
||||
const root = paths.dirname(path);
|
||||
const filename = paths.basename(path);
|
||||
if (!AUTH_CONFIG_FILENAME.test(filename) || paths.join(root, filename) !== path) throw invalid();
|
||||
const response = await request({ version: PROTOCOL_VERSION, operation: "read-local-users", root, filename });
|
||||
if (Object.keys(response).some((key) => !["version", "ok", "found", "contentBase64"].includes(key))) throw invalid();
|
||||
const contents = contentFrom(response, MAX_AUTH_CONFIG_BYTES);
|
||||
if (contents === undefined) throw invalid();
|
||||
return contents;
|
||||
},
|
||||
async create(root, directory, filename, contents) {
|
||||
if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > directoryMaximum(directory)) throw invalid();
|
||||
const response = await request(recordRequest("create", root, directory, filename, contents));
|
||||
if (response.created === undefined) throw invalid();
|
||||
return response.created;
|
||||
},
|
||||
async read(root, directory, filename) {
|
||||
return contentFrom(await request(recordRequest("read", root, directory, filename)), directoryMaximum(directory));
|
||||
},
|
||||
async replace(root, directory, filename, contents) {
|
||||
if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > directoryMaximum(directory)) throw invalid();
|
||||
const response = await request(recordRequest("replace", root, directory, filename, contents));
|
||||
if (response.replaced !== true) throw invalid();
|
||||
},
|
||||
async remove(root, directory, filename) {
|
||||
const response = await request(recordRequest("remove", root, directory, filename));
|
||||
return response.removed === true;
|
||||
},
|
||||
async list(root, directory, maximumEntries = DEFAULT_MAX_ENTRIES) {
|
||||
if (!Number.isInteger(maximumEntries) || maximumEntries < 1 || maximumEntries > MAX_ENTRIES) throw invalid();
|
||||
const response = await request({
|
||||
version: PROTOCOL_VERSION,
|
||||
operation: "list",
|
||||
root,
|
||||
directory,
|
||||
maximumEntries,
|
||||
});
|
||||
if (response.more !== undefined) throw invalid();
|
||||
return listedEntries(response, directory, maximumEntries);
|
||||
},
|
||||
async listPage(root, directory, afterName, maximumEntries) {
|
||||
if (directory !== "sessions" || !Number.isInteger(maximumEntries)
|
||||
|| maximumEntries < 1 || maximumEntries > MAX_ENTRIES
|
||||
|| (afterName !== undefined && !DIGEST_FILENAME.test(afterName))) throw invalid();
|
||||
const response = await request({
|
||||
version: PROTOCOL_VERSION,
|
||||
operation: "list",
|
||||
root,
|
||||
directory,
|
||||
maximumEntries,
|
||||
continuation: true,
|
||||
...(afterName === undefined ? {} : { afterName }),
|
||||
});
|
||||
if (response.more === undefined) throw invalid();
|
||||
const entries = listedEntries(response, directory, maximumEntries);
|
||||
let previous = afterName;
|
||||
for (const entry of entries) {
|
||||
if (previous !== undefined && entry.name <= previous) throw invalid();
|
||||
previous = entry.name;
|
||||
}
|
||||
if (response.more && entries.length !== maximumEntries) throw invalid();
|
||||
if (response.more && (previous === undefined || previous === afterName)) throw invalid();
|
||||
return { entries, more: response.more };
|
||||
},
|
||||
async claimConsume(root, filename) {
|
||||
return contentFrom(await request(recordRequest("claim-consume", root, "oidc", filename)), MAX_OIDC_BYTES);
|
||||
},
|
||||
async readClaim(root, filename) {
|
||||
return contentFrom(await request(recordRequest("read-claim", root, "oidc", filename)), MAX_OIDC_BYTES);
|
||||
},
|
||||
async removeClaim(root, filename) {
|
||||
const response = await request(recordRequest("remove-claim", root, "oidc", filename));
|
||||
return response.removed === true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWindowsAuthStorageBridge(options: WindowsAuthStorageBridgeOptions = {}): WindowsAuthStorageBridge {
|
||||
return createAuthStorageBridge("windows", options);
|
||||
}
|
||||
|
||||
/** POSIX uses the same single hidden tht protocol and bounds, with native canonical path rules. */
|
||||
export function createPosixAuthStorageBridge(options: WindowsAuthStorageBridgeOptions = {}): WindowsAuthStorageBridge {
|
||||
return createAuthStorageBridge("posix", options);
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
import type { RpcClient } from "../rpc/rpc-client.js";
|
||||
|
||||
const GENERIC_MODEL_FAILURE =
|
||||
"Model request failed. Check provider connectivity, then Resume the session.";
|
||||
const SUBSCRIPTION_MODEL_FAILURE =
|
||||
"The selected model is unavailable for the current subscription. Choose another model and start a new session.";
|
||||
const PHASE_STARTED_NOTIFICATION_PREFIX = "__tht_phase_started__:";
|
||||
|
||||
function phaseStartedNotification(message: unknown): string | null {
|
||||
if (typeof message !== "string" || !message.startsWith(PHASE_STARTED_NOTIFICATION_PREFIX)) {
|
||||
return null;
|
||||
}
|
||||
const phase = message.slice(PHASE_STARTED_NOTIFICATION_PREFIX.length);
|
||||
return /^F[1-8]$/.test(phase) ? phase : "";
|
||||
}
|
||||
|
||||
function safeModelFailure(error: unknown): string {
|
||||
const detail = typeof error === "string" ? error : "";
|
||||
const isSubscriptionFailure =
|
||||
/\b429\b/.test(detail) &&
|
||||
/(subscription plan|code["':\s]+1311|does not yet include access)/i.test(detail);
|
||||
|
||||
return isSubscriptionFailure ? SUBSCRIPTION_MODEL_FAILURE : GENERIC_MODEL_FAILURE;
|
||||
}
|
||||
|
||||
export type ToolActivity = {
|
||||
kind: "tool";
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
status: "running" | "completed" | "failed";
|
||||
};
|
||||
|
||||
export type TokenUsage = {
|
||||
input: number;
|
||||
cacheRead: number;
|
||||
output: number;
|
||||
totalTokens: number;
|
||||
contextWindow: number;
|
||||
};
|
||||
|
||||
export type ClientEvent =
|
||||
| { type: "ui_request"; ui_request: any }
|
||||
| { type: "text_delta"; text: string }
|
||||
| { type: "activity_delta"; text: string }
|
||||
| { type: "activity_event"; activity: ToolActivity }
|
||||
| { type: "usage"; usage: TokenUsage }
|
||||
| { type: "info"; [k: string]: any }
|
||||
| { type: "system_event"; event: string; phase?: string };
|
||||
|
||||
export type TurnState = "idle" | "running" | "waiting" | "failed";
|
||||
|
||||
export class SessionBridge {
|
||||
private state: TurnState = "idle";
|
||||
private pending: any = null;
|
||||
// Pi (rpc-mode createDialogPromise) assegna a ogni ctx.ui.input un id RPC PROPRIO
|
||||
// (crypto.randomUUID) e correla extension_ui_response su quell'id — NON sull'id interno
|
||||
// del descriptor (che viaggia opaco nel `title`). Va memorizzato e rimandato indietro,
|
||||
// altrimenti Pi scarta la risposta e ctx.ui.input non si risolve mai (stuck senza output).
|
||||
private pendingPiId: string | null = null;
|
||||
private contextWindow = 0;
|
||||
private cbs = new Set<(e: ClientEvent) => void>();
|
||||
|
||||
constructor(private rpc: RpcClient) {
|
||||
rpc.on("event", (m) => {
|
||||
if (m.type === "extension_ui_request" && m.method === "input") {
|
||||
let descriptor: unknown;
|
||||
try { descriptor = JSON.parse(m.title as string); } catch { return; }
|
||||
this.pending = descriptor;
|
||||
this.pendingPiId = m.id as string | null;
|
||||
this.state = "waiting";
|
||||
this.fan({ type: "ui_request", ui_request: descriptor });
|
||||
} else if (m.type === "message_end" && m.message?.role === "assistant") {
|
||||
this.emitUsage(m.message.usage);
|
||||
if (m.message.stopReason === "error") {
|
||||
this.markFailed();
|
||||
this.fan({
|
||||
type: "info",
|
||||
level: "error",
|
||||
text: safeModelFailure(m.message.errorMessage),
|
||||
});
|
||||
}
|
||||
} else if (m.type === "extension_ui_request" && m.method === "notify") {
|
||||
const phase = phaseStartedNotification(m.message);
|
||||
if (phase) this.fan({ type: "system_event", event: "phase_started", phase });
|
||||
else if (phase === null) {
|
||||
this.fan({ type: "info", level: m.notifyType ?? "info", text: m.message ?? "" });
|
||||
}
|
||||
} else if (m.type === "message_update" && m.assistantMessageEvent?.type === "text_delta") {
|
||||
this.fan({ type: "text_delta", text: m.assistantMessageEvent.delta ?? "" });
|
||||
} else if (m.type === "message_update" && m.assistantMessageEvent?.type === "thinking_delta") {
|
||||
this.fan({ type: "activity_delta", text: m.assistantMessageEvent.delta ?? "" });
|
||||
} else if (m.type === "text_delta") {
|
||||
this.fan({ type: "text_delta", text: m.text ?? "" });
|
||||
} else if (m.type === "tool_execution_start") {
|
||||
this.emitToolActivity(m, "start");
|
||||
} else if (m.type === "tool_execution_end") {
|
||||
this.emitToolActivity(m, "end");
|
||||
// Tool updates and raw payloads remain intentionally dropped.
|
||||
} else if (m.type === "system_event") {
|
||||
if (typeof m.event === "string" && m.event.trim() !== "") {
|
||||
this.fan({ type: "system_event", event: m.event });
|
||||
}
|
||||
} else if (m.type === "agent_end") {
|
||||
if (this.state !== "failed" && !this.pending) this.state = "idle";
|
||||
this.fan({ type: "system_event", event: "agent_end" });
|
||||
} else if (m.type === "agent_start") {
|
||||
this.state = "running";
|
||||
this.fan({ type: "system_event", event: "agent_start" });
|
||||
} else if (m.type === "turn_end") {
|
||||
this.fan({ type: "system_event", event: "turn_end" });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private emitToolActivity(message: any, lifecycle: "start" | "end"): void {
|
||||
const toolCallId = message.toolCallId;
|
||||
if (typeof toolCallId !== "string" || toolCallId.trim() === "") return;
|
||||
|
||||
const toolName =
|
||||
typeof message.toolName === "string" && message.toolName.trim() !== ""
|
||||
? message.toolName
|
||||
: "Tool";
|
||||
const status =
|
||||
lifecycle === "start" ? "running" : message.isError === true ? "failed" : "completed";
|
||||
|
||||
this.fan({
|
||||
type: "activity_event",
|
||||
activity: { kind: "tool", toolCallId, toolName, status },
|
||||
});
|
||||
}
|
||||
|
||||
private emitUsage(usage: any): void {
|
||||
if (!usage || this.contextWindow <= 0) return;
|
||||
const number = (value: unknown): number =>
|
||||
typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
|
||||
const input = number(usage.input);
|
||||
const cacheRead = number(usage.cacheRead);
|
||||
const output = number(usage.output);
|
||||
const reportedTotal = number(usage.totalTokens);
|
||||
this.fan({
|
||||
type: "usage",
|
||||
usage: {
|
||||
input,
|
||||
cacheRead,
|
||||
output,
|
||||
totalTokens: reportedTotal || input + cacheRead + output,
|
||||
contextWindow: this.contextWindow,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private fan(e: ClientEvent) { for (const cb of this.cbs) cb(e); }
|
||||
|
||||
turnState(): TurnState { return this.state; }
|
||||
|
||||
beginTurn(): void { this.state = "running"; }
|
||||
|
||||
setContextWindow(value: unknown): void {
|
||||
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
|
||||
this.contextWindow = value;
|
||||
}
|
||||
}
|
||||
|
||||
/** Record a backend-detected failure; callers own any sanitized client message. */
|
||||
markFailed(): void { this.state = "failed"; }
|
||||
|
||||
onClientEvent(cb: (e: ClientEvent) => void): void { this.cbs.add(cb); }
|
||||
|
||||
/** Eventi generati dal backend stesso (es. exit inatteso del child Pi), non da Pi. */
|
||||
emitClientEvent(e: ClientEvent): void { this.fan(e); }
|
||||
|
||||
/**
|
||||
* Deliver a reviewer response to Pi. Accepts ONLY a response matching the pending
|
||||
* descriptor: a stale/duplicate response (old gate id, double submit) would otherwise
|
||||
* be sent with the CURRENT gate's RPC id, flip the state to running, and leave the
|
||||
* real gate waiting. Returns false when rejected so the route can 409.
|
||||
*/
|
||||
respond(uiResponse: object & { id: string }): boolean {
|
||||
if (!this.pending || uiResponse.id !== (this.pending as { id?: unknown }).id) return false;
|
||||
// Correla sull'id RPC di Pi; `value` porta l'uiResponse (con l'id del descriptor) cosi'
|
||||
// il check interno del gate (resp.id === descriptor.id) regge.
|
||||
const piId = this.pendingPiId ?? uiResponse.id;
|
||||
this.state = "running";
|
||||
this.rpc.send({ type: "extension_ui_response", id: piId, value: JSON.stringify(uiResponse) });
|
||||
this.pending = null;
|
||||
this.pendingPiId = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
steer(text: string): void {
|
||||
this.state = "running";
|
||||
this.rpc.send({ type: "steer", message: text });
|
||||
}
|
||||
|
||||
pendingWidget(): object | null { return this.pending; }
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,252 +0,0 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||
import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js";
|
||||
|
||||
const MAX_SOURCE_ROWS = 5;
|
||||
const MAX_REPRESENTATIVE_VALUES = 5;
|
||||
const MAX_SOURCE_COLUMNS_PER_TARGET = 8;
|
||||
const MAX_SOURCE_VALUE_BYTES = 256;
|
||||
|
||||
export type DescriptionSourceSampleValue = string | number | boolean | null;
|
||||
|
||||
export interface DescriptionSourceSampleField {
|
||||
name: string;
|
||||
value: DescriptionSourceSampleValue;
|
||||
}
|
||||
|
||||
export interface DescriptionSourceSampleRow {
|
||||
fields: readonly DescriptionSourceSampleField[];
|
||||
}
|
||||
|
||||
export interface DescriptionSourceRepresentativeValues {
|
||||
column: string;
|
||||
values: readonly Exclude<DescriptionSourceSampleValue, null>[];
|
||||
}
|
||||
|
||||
export interface DescriptionTargetSourceSample {
|
||||
targetId: string;
|
||||
tableName: string;
|
||||
rows: readonly DescriptionSourceSampleRow[];
|
||||
representativeValues: readonly DescriptionSourceRepresentativeValues[];
|
||||
}
|
||||
|
||||
export interface DescriptionSourceSamplingTarget {
|
||||
targetId: string;
|
||||
tableName: string;
|
||||
columnNames: readonly string[];
|
||||
}
|
||||
|
||||
/** Optional, transient source context for one model-completion batch. */
|
||||
export interface DescriptionSourceSampler {
|
||||
sample(
|
||||
database: WorkspaceDatabase,
|
||||
targets: readonly DescriptionSourceSamplingTarget[],
|
||||
signal: AbortSignal,
|
||||
): Promise<readonly DescriptionTargetSourceSample[]>;
|
||||
}
|
||||
|
||||
function quoteIdentifier(identifier: string): string {
|
||||
return `"${identifier.replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
function boundedUtf8(value: string, maxBytes: number): string {
|
||||
const normalized = value
|
||||
.normalize("NFC")
|
||||
.replace(/\r\n?/g, "\n")
|
||||
.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ");
|
||||
if (Buffer.byteLength(normalized, "utf8") <= maxBytes) return normalized;
|
||||
let result = "";
|
||||
let bytes = 0;
|
||||
for (const character of normalized) {
|
||||
const characterBytes = Buffer.byteLength(character, "utf8");
|
||||
if (bytes + characterBytes > maxBytes) break;
|
||||
result += character;
|
||||
bytes += characterBytes;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeValue(value: unknown): DescriptionSourceSampleValue | undefined {
|
||||
if (value === null) return null;
|
||||
if (typeof value === "string") return boundedUtf8(value, MAX_SOURCE_VALUE_BYTES);
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "number") return Number.isFinite(value) ? value : undefined;
|
||||
if (typeof value === "bigint") return boundedUtf8(String(value), MAX_SOURCE_VALUE_BYTES);
|
||||
if (value instanceof Date && !Number.isNaN(value.valueOf())) return value.toISOString();
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function distinctKey(value: Exclude<DescriptionSourceSampleValue, null>): string {
|
||||
return `${typeof value}:${String(value)}`;
|
||||
}
|
||||
|
||||
function columnsFor(target: DescriptionSourceSamplingTarget): string[] {
|
||||
return [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
|
||||
}
|
||||
|
||||
function normalizedSample(
|
||||
target: DescriptionSourceSamplingTarget,
|
||||
columnNames: readonly string[],
|
||||
sourceRows: readonly Record<string, unknown>[],
|
||||
): DescriptionTargetSourceSample {
|
||||
const rows = sourceRows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
|
||||
fields: columnNames.flatMap((name) => {
|
||||
const value = normalizeValue(row[name]);
|
||||
return value === undefined ? [] : [{ name, value }];
|
||||
}),
|
||||
}));
|
||||
const valuesByColumn = new Map<string, Exclude<DescriptionSourceSampleValue, null>[]>();
|
||||
const seenByColumn = new Map<string, Set<string>>();
|
||||
let representativeValueCount = 0;
|
||||
for (const row of rows) {
|
||||
for (const field of row.fields) {
|
||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||
if (field.value === null) continue;
|
||||
const seen = seenByColumn.get(field.name) ?? new Set<string>();
|
||||
const key = distinctKey(field.value);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
seenByColumn.set(field.name, seen);
|
||||
const values = valuesByColumn.get(field.name) ?? [];
|
||||
values.push(field.value);
|
||||
valuesByColumn.set(field.name, values);
|
||||
representativeValueCount += 1;
|
||||
}
|
||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||
}
|
||||
return {
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows,
|
||||
representativeValues: columnNames.flatMap((column) => {
|
||||
const values = valuesByColumn.get(column);
|
||||
return values && values.length > 0 ? [{ column, values }] : [];
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function samplingSql(
|
||||
database: WorkspaceDatabase,
|
||||
target: DescriptionSourceSamplingTarget,
|
||||
columns: readonly string[],
|
||||
): string {
|
||||
const projections = columns.map((columnName) => {
|
||||
const identifier = quoteIdentifier(columnName);
|
||||
return `LEFT((${identifier})::text, ${MAX_SOURCE_VALUE_BYTES}) AS ${identifier}`;
|
||||
});
|
||||
return [
|
||||
`SELECT ${projections.join(", ")}`,
|
||||
`FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`,
|
||||
`LIMIT ${MAX_SOURCE_ROWS}`,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
/** Bounded source sampler that follows the database's PostgreSQL-wire or REST binding. */
|
||||
export class ConcreteDescriptionSourceSampler implements DescriptionSourceSampler {
|
||||
constructor(
|
||||
private readonly access: CatalogPostgresAccess,
|
||||
private readonly secretStore?: Pick<WorkspaceSecretStore, "materialize">,
|
||||
) {}
|
||||
|
||||
async sample(
|
||||
database: WorkspaceDatabase,
|
||||
targets: readonly DescriptionSourceSamplingTarget[],
|
||||
signal: AbortSignal,
|
||||
): Promise<readonly DescriptionTargetSourceSample[]> {
|
||||
if (database.binding.transport === "rest_api") {
|
||||
return await this.sampleRest(database, targets, signal);
|
||||
}
|
||||
const client = await this.access.connect(database, signal);
|
||||
let transactionOpen = false;
|
||||
try {
|
||||
await client.query("BEGIN TRANSACTION READ ONLY", []);
|
||||
transactionOpen = true;
|
||||
const samples: DescriptionTargetSourceSample[] = [];
|
||||
for (const target of targets) {
|
||||
const columnNames = columnsFor(target);
|
||||
if (columnNames.length === 0) {
|
||||
samples.push({
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows: [],
|
||||
representativeValues: [],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const projections = columnNames.map((columnName) => {
|
||||
const identifier = quoteIdentifier(columnName);
|
||||
return `LEFT((${identifier})::text, $1) AS ${identifier}`;
|
||||
});
|
||||
const sql = [
|
||||
`SELECT ${projections.join(", ")}`,
|
||||
`FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`,
|
||||
"LIMIT $2",
|
||||
].join(" ");
|
||||
const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]);
|
||||
samples.push(normalizedSample(target, columnNames, result.rows));
|
||||
}
|
||||
return samples;
|
||||
} finally {
|
||||
if (transactionOpen) await client.query("ROLLBACK", []).catch(() => undefined);
|
||||
await client.end().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
private async sampleRest(
|
||||
database: WorkspaceDatabase,
|
||||
targets: readonly DescriptionSourceSamplingTarget[],
|
||||
signal: AbortSignal,
|
||||
): Promise<readonly DescriptionTargetSourceSample[]> {
|
||||
if (!this.secretStore) throw new CatalogConnectorError("REST source sampling is not configured");
|
||||
const auth = database.binding.restAuth ?? "bearer";
|
||||
const materialized = this.secretStore.materialize(
|
||||
database.workspaceId,
|
||||
auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
|
||||
);
|
||||
try {
|
||||
const headers: Record<string, string> = { "content-type": "application/json" };
|
||||
if (auth !== "none") {
|
||||
const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
|
||||
if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
|
||||
const credential = (await readFile(credentialFile, "utf8")).trim();
|
||||
if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
|
||||
else headers["x-api-key"] = credential;
|
||||
}
|
||||
const baseUrl = database.binding.baseUrl?.replace(/\/+$/, "");
|
||||
if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete");
|
||||
const samples: DescriptionTargetSourceSample[] = [];
|
||||
for (const target of targets) {
|
||||
const columnNames = columnsFor(target);
|
||||
if (columnNames.length === 0) {
|
||||
samples.push(normalizedSample(target, columnNames, []));
|
||||
continue;
|
||||
}
|
||||
const response = await fetch(`${baseUrl}/rpc/run_query`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ query_text: samplingSql(database, target, columnNames) }),
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) throw new CatalogConnectorError("REST source sampling failed");
|
||||
const body: unknown = await response.json();
|
||||
if (!Array.isArray(body)
|
||||
|| body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
|
||||
throw new CatalogConnectorError("REST source sampling response is invalid");
|
||||
}
|
||||
samples.push(normalizedSample(
|
||||
target,
|
||||
columnNames,
|
||||
body as Array<Record<string, unknown>>,
|
||||
));
|
||||
}
|
||||
return samples;
|
||||
} catch (error) {
|
||||
if (error instanceof CatalogConnectorError) throw error;
|
||||
throw new CatalogConnectorError("REST source sampling failed");
|
||||
} finally {
|
||||
materialized.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
import type { CatalogRelationship, CatalogRepository } from "./types.js";
|
||||
|
||||
export interface EffectiveRelationshipSnapshotReader {
|
||||
list(databaseId: string): Promise<CatalogRelationship[]>;
|
||||
}
|
||||
|
||||
export interface EffectiveRelationshipSnapshotCoordinator {
|
||||
run<T>(databaseId: string, operation: () => Promise<T>): Promise<T>;
|
||||
}
|
||||
|
||||
export class EffectiveRelationshipSnapshotStaleError extends Error {}
|
||||
|
||||
interface EffectiveRelationship {
|
||||
sourceTable: string;
|
||||
sourceColumns: string[];
|
||||
targetTable: string;
|
||||
targetColumns: string[];
|
||||
origin: CatalogRelationship["origin"];
|
||||
}
|
||||
|
||||
interface EffectiveRelationshipSnapshot {
|
||||
schemaVersion: 1;
|
||||
workspaceId: string;
|
||||
relationships: EffectiveRelationship[];
|
||||
}
|
||||
|
||||
const originRank: Record<CatalogRelationship["origin"], number> = {
|
||||
physical: 0,
|
||||
manual: 1,
|
||||
generated: 2,
|
||||
};
|
||||
|
||||
function endpointKey(relationship: EffectiveRelationship): string {
|
||||
return [
|
||||
relationship.sourceTable,
|
||||
relationship.sourceColumns.join("\u0000"),
|
||||
relationship.targetTable,
|
||||
relationship.targetColumns.join("\u0000"),
|
||||
].join("\u0001");
|
||||
}
|
||||
|
||||
function compareRelationships(left: EffectiveRelationship, right: EffectiveRelationship): number {
|
||||
return endpointKey(left).localeCompare(endpointKey(right))
|
||||
|| originRank[left.origin] - originRank[right.origin];
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapter from the mutable Catalog model to the immutable relationship contract consumed by the
|
||||
* harness. The returned JSON is a deterministic projection, never an authored second store.
|
||||
*/
|
||||
export class EffectiveRelationshipSnapshotProvider {
|
||||
constructor(
|
||||
private readonly repository: Pick<CatalogRepository, "get" | "getByWorkspace">,
|
||||
private readonly relationships: EffectiveRelationshipSnapshotReader,
|
||||
private readonly operations: EffectiveRelationshipSnapshotCoordinator,
|
||||
) {}
|
||||
|
||||
async render(workspaceId: string): Promise<string | undefined> {
|
||||
const database = await this.repository.getByWorkspace(workspaceId);
|
||||
if (!database) return undefined;
|
||||
return await this.operations.run(database.id, async () => {
|
||||
const current = await this.repository.get(database.id);
|
||||
if (!current || current.schemaSyncedVersion !== current.version) {
|
||||
throw new EffectiveRelationshipSnapshotStaleError(
|
||||
"effective relationship snapshot requires a current full schema synchronization",
|
||||
);
|
||||
}
|
||||
const projected = (await this.relationships.list(database.id))
|
||||
.filter((relationship) => relationship.status === "active")
|
||||
.map((relationship): EffectiveRelationship => ({
|
||||
sourceTable: relationship.sourceTableName,
|
||||
sourceColumns: relationship.columns.map((column) => column.sourceColumnName),
|
||||
targetTable: relationship.targetTableName,
|
||||
targetColumns: relationship.columns.map((column) => column.targetColumnName),
|
||||
origin: relationship.origin,
|
||||
}))
|
||||
.sort(compareRelationships);
|
||||
|
||||
const seen = new Set<string>();
|
||||
const snapshot: EffectiveRelationshipSnapshot = {
|
||||
schemaVersion: 1,
|
||||
workspaceId,
|
||||
relationships: projected.filter((relationship) => {
|
||||
const key = endpointKey(relationship);
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
}),
|
||||
};
|
||||
return `${JSON.stringify(snapshot, null, 2)}\n`;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,260 +0,0 @@
|
||||
import type {
|
||||
CatalogLogicalRelationship,
|
||||
CatalogLogicalRelationshipCandidate,
|
||||
CatalogLogicalRelationshipContext,
|
||||
CatalogLogicalRelationshipEndpoint,
|
||||
CatalogRelationship,
|
||||
CatalogRepository,
|
||||
} from "./types.js";
|
||||
|
||||
export class LogicalRelationshipDatabaseNotFoundError extends Error {}
|
||||
export class LogicalRelationshipDuplicateError extends Error {}
|
||||
export class LogicalRelationshipNotFoundError extends Error {}
|
||||
export class LogicalRelationshipReadOnlyError extends Error {}
|
||||
export class LogicalRelationshipSchemaStaleError extends Error {}
|
||||
export class LogicalRelationshipTargetNotUniqueError extends Error {}
|
||||
export class LogicalRelationshipTypeIncompatibleError extends Error {}
|
||||
export class LogicalRelationshipColumnNotFoundError extends Error {
|
||||
constructor(readonly field: "sourceColumnId" | "targetColumnId") {
|
||||
super(`Catalog column '${field}' was not found`);
|
||||
}
|
||||
}
|
||||
|
||||
export interface RebuildGeneratedRelationshipsResult {
|
||||
added: number;
|
||||
alreadyPresent: number;
|
||||
excluded: number;
|
||||
ambiguous: number;
|
||||
}
|
||||
|
||||
function identifierTokens(value: string): string[] {
|
||||
return value
|
||||
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function singularWord(value: string): string {
|
||||
if (value.length > 4 && value.endsWith("ies")) return `${value.slice(0, -3)}y`;
|
||||
if (value.length > 4 && /(ches|shes|xes|zes|ses)$/.test(value)) return value.slice(0, -2);
|
||||
if (value.length > 3 && value.endsWith("s") && !/(ss|us)$/.test(value)) return value.slice(0, -1);
|
||||
return value;
|
||||
}
|
||||
|
||||
function tableAliases(tableName: string): string[] {
|
||||
const tokens = identifierTokens(tableName);
|
||||
if (tokens.length === 0) return [];
|
||||
const normalized = tokens.join("_");
|
||||
const singular = [...tokens];
|
||||
singular[singular.length - 1] = singularWord(singular[singular.length - 1]);
|
||||
return [...new Set([normalized, singular.join("_")])];
|
||||
}
|
||||
|
||||
const GENERIC_PRIMARY_KEY_NAMES = new Set(["id", "key", "code", "pk"]);
|
||||
|
||||
function nameMatches(
|
||||
source: CatalogLogicalRelationshipEndpoint,
|
||||
target: CatalogLogicalRelationshipEndpoint,
|
||||
): boolean {
|
||||
const sourceName = identifierTokens(source.columnName).join("_");
|
||||
const targetName = identifierTokens(target.columnName).join("_");
|
||||
if (!sourceName || !targetName) return false;
|
||||
const expected = new Set<string>();
|
||||
if (!GENERIC_PRIMARY_KEY_NAMES.has(targetName)) expected.add(targetName);
|
||||
for (const alias of tableAliases(target.tableName)) {
|
||||
expected.add(`${alias}_${targetName}`);
|
||||
expected.add(`${alias.replaceAll("_", "")}${targetName.replaceAll("_", "")}`);
|
||||
if (targetName === "id" || targetName === "pk") expected.add(alias);
|
||||
}
|
||||
return expected.has(sourceName);
|
||||
}
|
||||
|
||||
function canonicalDataType(value: string): string {
|
||||
const normalized = value.trim().toLowerCase().replace(/\s+/g, " ");
|
||||
const arraySuffix = normalized.endsWith("[]") ? "[]" : "";
|
||||
const base = arraySuffix ? normalized.slice(0, -2) : normalized;
|
||||
const withoutModifier = base.replace(/\([^)]*\)/g, "").trim();
|
||||
const aliases: Record<string, string> = {
|
||||
int2: "smallint",
|
||||
smallserial: "smallint",
|
||||
int4: "integer",
|
||||
int: "integer",
|
||||
serial: "integer",
|
||||
int8: "bigint",
|
||||
bigserial: "bigint",
|
||||
decimal: "numeric",
|
||||
varchar: "text",
|
||||
"character varying": "text",
|
||||
bool: "boolean",
|
||||
"timestamp without time zone": "timestamp",
|
||||
"timestamp with time zone": "timestamptz",
|
||||
"time without time zone": "time",
|
||||
"time with time zone": "timetz",
|
||||
};
|
||||
return `${aliases[withoutModifier] ?? withoutModifier}${arraySuffix}`;
|
||||
}
|
||||
|
||||
function typesCompatible(left: string, right: string): boolean {
|
||||
return canonicalDataType(left) === canonicalDataType(right);
|
||||
}
|
||||
|
||||
function pairKey(sourceColumnId: string, targetColumnId: string): string {
|
||||
return `${sourceColumnId}\u0000${targetColumnId}`;
|
||||
}
|
||||
|
||||
function relationshipPair(relationship: CatalogLogicalRelationship): CatalogLogicalRelationshipCandidate {
|
||||
return {
|
||||
sourceColumnId: relationship.columns[0].sourceColumnId,
|
||||
targetColumnId: relationship.columns[0].targetColumnId,
|
||||
};
|
||||
}
|
||||
|
||||
function relationshipSortKey(relationship: CatalogRelationship): string {
|
||||
const sourceColumns = relationship.columns.map((column) => column.sourceColumnName).join(",");
|
||||
const targetColumns = relationship.columns.map((column) => column.targetColumnName).join(",");
|
||||
return [
|
||||
relationship.sourceTableName,
|
||||
sourceColumns,
|
||||
relationship.targetTableName,
|
||||
targetColumns,
|
||||
relationship.origin,
|
||||
].join("\u0000");
|
||||
}
|
||||
|
||||
export class CatalogLogicalRelationshipService {
|
||||
constructor(private readonly repository: CatalogRepository) {}
|
||||
|
||||
async list(databaseId: string): Promise<CatalogRelationship[]> {
|
||||
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
|
||||
const relationships: CatalogRelationship[] = [
|
||||
...await this.repository.listRelationships(databaseId),
|
||||
...await this.repository.listLogicalRelationships(databaseId),
|
||||
];
|
||||
return relationships.sort((left, right) => relationshipSortKey(left).localeCompare(relationshipSortKey(right)));
|
||||
}
|
||||
|
||||
async addManual(
|
||||
databaseId: string,
|
||||
sourceColumnId: string,
|
||||
targetColumnId: string,
|
||||
): Promise<CatalogLogicalRelationship> {
|
||||
const context = await this.requiredContext(databaseId);
|
||||
const source = context.endpoints.find((endpoint) => endpoint.columnId === sourceColumnId);
|
||||
if (!source) throw new LogicalRelationshipColumnNotFoundError("sourceColumnId");
|
||||
const target = context.endpoints.find((endpoint) => endpoint.columnId === targetColumnId);
|
||||
if (!target) throw new LogicalRelationshipColumnNotFoundError("targetColumnId");
|
||||
if (sourceColumnId === targetColumnId
|
||||
|| target.primaryKeyPosition === null
|
||||
|| target.tablePrimaryKeyColumnCount !== 1) {
|
||||
throw new LogicalRelationshipTargetNotUniqueError();
|
||||
}
|
||||
if (!typesCompatible(source.dataType, target.dataType)) {
|
||||
throw new LogicalRelationshipTypeIncompatibleError();
|
||||
}
|
||||
const key = pairKey(sourceColumnId, targetColumnId);
|
||||
if (context.physicalPairs.some((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId) === key)
|
||||
|| context.logicalRelationships.some((relationship) => {
|
||||
const pair = relationshipPair(relationship);
|
||||
return pairKey(pair.sourceColumnId, pair.targetColumnId) === key;
|
||||
})) throw new LogicalRelationshipDuplicateError();
|
||||
const created = await this.repository.insertLogicalRelationship(
|
||||
databaseId,
|
||||
sourceColumnId,
|
||||
targetColumnId,
|
||||
false,
|
||||
);
|
||||
if (!created) throw new LogicalRelationshipDuplicateError();
|
||||
return created;
|
||||
}
|
||||
|
||||
async rebuildGenerated(databaseId: string): Promise<RebuildGeneratedRelationshipsResult> {
|
||||
const context = await this.requiredContext(databaseId);
|
||||
const targets = context.endpoints.filter((endpoint) => (
|
||||
endpoint.primaryKeyPosition !== null && endpoint.tablePrimaryKeyColumnCount === 1
|
||||
));
|
||||
const physical = new Set(context.physicalPairs.map((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId)));
|
||||
const active = new Set<string>();
|
||||
const excluded = new Set<string>();
|
||||
for (const relationship of context.logicalRelationships) {
|
||||
const pair = relationshipPair(relationship);
|
||||
(relationship.status === "excluded" ? excluded : active)
|
||||
.add(pairKey(pair.sourceColumnId, pair.targetColumnId));
|
||||
}
|
||||
|
||||
const pending: CatalogLogicalRelationshipCandidate[] = [];
|
||||
let alreadyPresent = 0;
|
||||
let excludedCount = 0;
|
||||
let ambiguous = 0;
|
||||
const dimTimeTargets = targets.filter((target) => (
|
||||
identifierTokens(target.tableName).join("_") === "dim_time"
|
||||
));
|
||||
const sources = context.endpoints.filter((endpoint) => (
|
||||
endpoint.primaryKeyPosition === null || endpoint.tablePrimaryKeyColumnCount > 1
|
||||
));
|
||||
for (const source of sources) {
|
||||
const sourceName = identifierTokens(source.columnName).join("_");
|
||||
const isTimeKey = sourceName.endsWith("time_key")
|
||||
&& identifierTokens(source.tableName).join("_") !== "dim_time";
|
||||
const candidates = isTimeKey ? dimTimeTargets : targets;
|
||||
const matches = candidates.filter((target) => (
|
||||
target.columnId !== source.columnId
|
||||
&& typesCompatible(source.dataType, target.dataType)
|
||||
&& (isTimeKey || nameMatches(source, target))
|
||||
));
|
||||
if (matches.length > 1) {
|
||||
ambiguous += 1;
|
||||
continue;
|
||||
}
|
||||
if (matches.length === 0) continue;
|
||||
const candidate = { sourceColumnId: source.columnId, targetColumnId: matches[0].columnId };
|
||||
const key = pairKey(candidate.sourceColumnId, candidate.targetColumnId);
|
||||
if (physical.has(key) || active.has(key)) {
|
||||
alreadyPresent += 1;
|
||||
} else if (excluded.has(key)) {
|
||||
excludedCount += 1;
|
||||
} else {
|
||||
pending.push(candidate);
|
||||
}
|
||||
}
|
||||
const added = await this.repository.insertGeneratedLogicalRelationships(databaseId, pending);
|
||||
alreadyPresent += pending.length - added;
|
||||
return { added, alreadyPresent, excluded: excludedCount, ambiguous };
|
||||
}
|
||||
|
||||
async setStatus(
|
||||
databaseId: string,
|
||||
relationshipId: string,
|
||||
status: CatalogLogicalRelationship["status"],
|
||||
): Promise<CatalogLogicalRelationship> {
|
||||
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
|
||||
const updated = await this.repository.setLogicalRelationshipStatus(databaseId, relationshipId, status);
|
||||
if (updated) return updated;
|
||||
await this.assertNotPhysical(databaseId, relationshipId);
|
||||
throw new LogicalRelationshipNotFoundError();
|
||||
}
|
||||
|
||||
async deletePermanently(databaseId: string, relationshipId: string): Promise<void> {
|
||||
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
|
||||
if (await this.repository.deleteLogicalRelationship(databaseId, relationshipId)) return;
|
||||
await this.assertNotPhysical(databaseId, relationshipId);
|
||||
throw new LogicalRelationshipNotFoundError();
|
||||
}
|
||||
|
||||
private async requiredContext(databaseId: string): Promise<CatalogLogicalRelationshipContext> {
|
||||
const database = await this.repository.get(databaseId);
|
||||
if (!database) throw new LogicalRelationshipDatabaseNotFoundError();
|
||||
if (database.schemaSyncedVersion !== database.version) {
|
||||
throw new LogicalRelationshipSchemaStaleError();
|
||||
}
|
||||
const context = await this.repository.getLogicalRelationshipContext(databaseId);
|
||||
if (!context) throw new LogicalRelationshipDatabaseNotFoundError();
|
||||
return context;
|
||||
}
|
||||
|
||||
private async assertNotPhysical(databaseId: string, relationshipId: string): Promise<void> {
|
||||
if ((await this.repository.listRelationships(databaseId)).some((relationship) => relationship.id === relationshipId)) {
|
||||
throw new LogicalRelationshipReadOnlyError();
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,235 +0,0 @@
|
||||
import {
|
||||
closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
|
||||
type Stats,
|
||||
} from "node:fs";
|
||||
import { parseAllDocuments } from "yaml";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
loadSecretBundle,
|
||||
METADATA_GENERATION_SECRET_KEYS,
|
||||
} from "../config/secret-bundle.js";
|
||||
|
||||
const MAX_INSTALLATION_BYTES = 1024 * 1024;
|
||||
const RUNTIME_INSTALLATION_FILE = "/run/thothii-installation/thothii-installation.yaml";
|
||||
const modelId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
|
||||
const apiKeyEnvironment = z.enum(METADATA_GENERATION_SECRET_KEYS);
|
||||
const endpointSchema = z.object({
|
||||
baseUrl: z.string().min(1).max(2048).refine((value) => {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return (url.protocol === "http:" || url.protocol === "https:")
|
||||
&& url.username === "" && url.password === "" && url.search === "" && url.hash === "";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}),
|
||||
apiVersion: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/).optional(),
|
||||
}).strict();
|
||||
const configuredModelSchema = z.object({
|
||||
id: modelId,
|
||||
label: z.string().min(1).max(128).refine((value) => value.trim() === value && !/\p{Cc}/u.test(value)),
|
||||
litellm: z.object({
|
||||
provider: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/),
|
||||
model: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/),
|
||||
disableThinking: z.literal(true).optional(),
|
||||
endpoint: endpointSchema.optional(),
|
||||
}).strict(),
|
||||
apiKeyEnv: apiKeyEnvironment.optional(),
|
||||
}).strict().superRefine((value, context) => {
|
||||
if (value.apiKeyEnv === undefined && value.litellm.endpoint === undefined) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["apiKeyEnv"],
|
||||
message: "keyless models require an explicit endpoint",
|
||||
});
|
||||
}
|
||||
if (value.litellm.disableThinking === true && value.litellm.endpoint === undefined) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["litellm", "disableThinking"],
|
||||
message: "thinking may be disabled only for an explicit endpoint",
|
||||
});
|
||||
}
|
||||
});
|
||||
const metadataGenerationSchema = z.object({
|
||||
default: modelId.optional(),
|
||||
models: z.array(configuredModelSchema).max(64).default([]),
|
||||
}).strict();
|
||||
const installationSchema = z.object({
|
||||
metadataGeneration: metadataGenerationSchema.optional(),
|
||||
}).passthrough();
|
||||
|
||||
export interface MetadataGenerationModelChoice {
|
||||
id: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface MetadataGenerationModelCatalog {
|
||||
models: MetadataGenerationModelChoice[];
|
||||
default: string | null;
|
||||
}
|
||||
|
||||
export interface ResolvedMetadataGenerationModel {
|
||||
readonly id: string;
|
||||
readonly provider: string;
|
||||
readonly model: string;
|
||||
readonly disableThinking?: true;
|
||||
readonly endpoint?: Readonly<{ baseUrl: string; apiVersion?: string }>;
|
||||
readonly apiKeyEnv?: string;
|
||||
readonly apiKey?: string;
|
||||
}
|
||||
|
||||
export class MetadataGenerationModelUnavailableError extends Error {
|
||||
constructor() {
|
||||
super("metadata-generation model is unavailable");
|
||||
this.name = "MetadataGenerationModelUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
/** The complete interface callers need: safe discovery plus fail-closed runtime resolution. */
|
||||
export interface MetadataGenerationModels {
|
||||
catalog(): MetadataGenerationModelCatalog;
|
||||
resolve(selection: string): ResolvedMetadataGenerationModel;
|
||||
}
|
||||
|
||||
class RestartLoadedMetadataGenerationModels implements MetadataGenerationModels {
|
||||
readonly #models: ReadonlyMap<string, ResolvedMetadataGenerationModel>;
|
||||
readonly #catalog: MetadataGenerationModelCatalog;
|
||||
|
||||
constructor(
|
||||
models: ReadonlyMap<string, ResolvedMetadataGenerationModel> = new Map(),
|
||||
defaultModel: string | null = null,
|
||||
choices: MetadataGenerationModelChoice[] = [],
|
||||
) {
|
||||
this.#models = models;
|
||||
this.#catalog = {
|
||||
models: choices.map((choice) => ({ ...choice })),
|
||||
default: defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
catalog(): MetadataGenerationModelCatalog {
|
||||
return {
|
||||
models: this.#catalog.models.map((choice) => ({ ...choice })),
|
||||
default: this.#catalog.default,
|
||||
};
|
||||
}
|
||||
|
||||
resolve(selection: string): ResolvedMetadataGenerationModel {
|
||||
const model = typeof selection === "string" ? this.#models.get(selection) : undefined;
|
||||
if (!model) throw new MetadataGenerationModelUnavailableError();
|
||||
return model;
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(message = "metadata-generation configuration is invalid"): Error {
|
||||
return new Error(message);
|
||||
}
|
||||
|
||||
function protectedInstallationStat(file: string, info: Stats): boolean {
|
||||
const mode = info.mode & 0o777;
|
||||
if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
|
||||
|| info.size < 1 || info.size > MAX_INSTALLATION_BYTES) return false;
|
||||
if (file === RUNTIME_INSTALLATION_FILE && info.uid === 0 && mode === 0o444) return true;
|
||||
return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600);
|
||||
}
|
||||
|
||||
function readProtectedInstallation(file: string): string {
|
||||
let descriptor: number | undefined;
|
||||
try {
|
||||
const before = lstatSync(file);
|
||||
if (!protectedInstallationStat(file, before)) throw new Error("unavailable");
|
||||
descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const opened = fstatSync(descriptor);
|
||||
if (!protectedInstallationStat(file, opened)
|
||||
|| before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("unavailable");
|
||||
const source = readFileSync(descriptor, "utf8");
|
||||
const after = fstatSync(descriptor);
|
||||
const current = lstatSync(file);
|
||||
if (!protectedInstallationStat(file, after) || !protectedInstallationStat(file, current)
|
||||
|| opened.dev !== after.dev || opened.ino !== after.ino
|
||||
|| opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("unavailable");
|
||||
return source;
|
||||
} finally {
|
||||
if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized below */ }
|
||||
}
|
||||
}
|
||||
|
||||
function readInstallation(file: string): unknown {
|
||||
try {
|
||||
const documents = parseAllDocuments(readProtectedInstallation(file), { uniqueKeys: true });
|
||||
if (documents.length !== 1) throw invalid("metadata-generation installation must contain one YAML document");
|
||||
const document = documents[0];
|
||||
if (document.errors.length > 0 || document.warnings.length > 0) {
|
||||
throw invalid("metadata-generation installation contains invalid YAML");
|
||||
}
|
||||
return document.toJSON();
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith("metadata-generation")) throw error;
|
||||
throw invalid("metadata-generation installation is unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
export function loadMetadataGenerationModels(options: {
|
||||
installationFile?: string;
|
||||
secretsFile?: string;
|
||||
}): MetadataGenerationModels {
|
||||
if (!options.installationFile) return new RestartLoadedMetadataGenerationModels();
|
||||
const installation = installationSchema.safeParse(readInstallation(options.installationFile));
|
||||
if (!installation.success) throw invalid();
|
||||
const configured = installation.data.metadataGeneration;
|
||||
if (!configured || configured.models.length === 0) {
|
||||
if (configured?.default !== undefined) throw invalid("metadata-generation default does not identify a configured model");
|
||||
return new RestartLoadedMetadataGenerationModels();
|
||||
}
|
||||
if (!configured.default) throw invalid("metadata-generation default is required when models are configured");
|
||||
|
||||
const seen = new Set<string>();
|
||||
for (const model of configured.models) {
|
||||
if (seen.has(model.id)) throw invalid(`metadata-generation model id "${model.id}" is duplicated`);
|
||||
seen.add(model.id);
|
||||
}
|
||||
if (!seen.has(configured.default)) {
|
||||
throw invalid(`metadata-generation default "${configured.default}" is not configured`);
|
||||
}
|
||||
const requiresSecrets = configured.models.some((model) => model.apiKeyEnv !== undefined);
|
||||
let secrets: ReadonlyMap<string, string> = new Map();
|
||||
if (requiresSecrets) {
|
||||
if (!options.secretsFile) throw invalid("metadata-generation keyed models require THT_SECRETS_FILE");
|
||||
try {
|
||||
secrets = loadSecretBundle(options.secretsFile);
|
||||
} catch {
|
||||
throw invalid("metadata-generation secrets are unavailable");
|
||||
}
|
||||
}
|
||||
const models = new Map<string, ResolvedMetadataGenerationModel>();
|
||||
for (const configuredModel of configured.models) {
|
||||
let apiKey: string | undefined;
|
||||
if (configuredModel.apiKeyEnv !== undefined) {
|
||||
apiKey = secrets.get(configuredModel.apiKeyEnv);
|
||||
if (!apiKey) {
|
||||
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is missing`);
|
||||
}
|
||||
if (apiKey.length > 16 * 1024 || /\s/u.test(apiKey)) {
|
||||
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is unusable`);
|
||||
}
|
||||
}
|
||||
models.set(configuredModel.id, Object.freeze({
|
||||
id: configuredModel.id,
|
||||
provider: configuredModel.litellm.provider,
|
||||
model: configuredModel.litellm.model,
|
||||
...(configuredModel.litellm.disableThinking === true ? { disableThinking: true as const } : {}),
|
||||
...(configuredModel.litellm.endpoint === undefined
|
||||
? {}
|
||||
: { endpoint: Object.freeze({ ...configuredModel.litellm.endpoint }) }),
|
||||
...(configuredModel.apiKeyEnv === undefined
|
||||
? {}
|
||||
: { apiKeyEnv: configuredModel.apiKeyEnv, apiKey }),
|
||||
}));
|
||||
}
|
||||
return new RestartLoadedMetadataGenerationModels(
|
||||
models,
|
||||
configured.default,
|
||||
configured.models.map(({ id, label }) => ({ id, label })),
|
||||
);
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
import type { CatalogMetrics } from "./types.js";
|
||||
|
||||
export interface CatalogMetricCounts {
|
||||
tables: number;
|
||||
columns: number;
|
||||
sensitiveColumns: number;
|
||||
relationships: number;
|
||||
describedTables: number;
|
||||
describedColumns: number;
|
||||
}
|
||||
|
||||
export function hasCatalogDescription(target: {
|
||||
description: string | null;
|
||||
generatedDescription: string | null;
|
||||
}): boolean {
|
||||
return Boolean(target.description?.trim() || target.generatedDescription?.trim());
|
||||
}
|
||||
|
||||
export function latestCatalogTimestamp(
|
||||
values: readonly (string | null | undefined)[],
|
||||
): string | null {
|
||||
let latest: number | undefined;
|
||||
for (const value of values) {
|
||||
if (!value) continue;
|
||||
const timestamp = Date.parse(value);
|
||||
if (!Number.isFinite(timestamp)) continue;
|
||||
latest = latest === undefined ? timestamp : Math.max(latest, timestamp);
|
||||
}
|
||||
return latest === undefined ? null : new Date(latest).toISOString();
|
||||
}
|
||||
|
||||
export function createCatalogMetrics(
|
||||
databaseId: string | undefined,
|
||||
counts: CatalogMetricCounts,
|
||||
updatedAt: string | null,
|
||||
): CatalogMetrics {
|
||||
const descriptionTargets = counts.tables + counts.columns;
|
||||
const describedTargets = counts.describedTables + counts.describedColumns;
|
||||
return {
|
||||
scope: databaseId === undefined ? "global" : "database",
|
||||
databaseId: databaseId ?? null,
|
||||
tables: counts.tables,
|
||||
columns: counts.columns,
|
||||
sensitiveColumns: counts.sensitiveColumns,
|
||||
relationships: counts.relationships,
|
||||
descriptionTargets,
|
||||
describedTargets,
|
||||
descriptionCoverage: descriptionTargets === 0
|
||||
? 0
|
||||
: Math.round((describedTargets / descriptionTargets) * 100),
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely";
|
||||
import { Migrator, type MigrationProvider } from "kysely/migration";
|
||||
import { Pool } from "pg";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { CatalogDatabase } from "./repository.js";
|
||||
import * as initialMigration from "./migrations/001_workspace_databases.js";
|
||||
import * as catalogTablesMigration from "./migrations/002_catalog_tables.js";
|
||||
import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_sync.js";
|
||||
import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
|
||||
import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
|
||||
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
|
||||
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
|
||||
import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
|
||||
|
||||
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
|
||||
const host = process.env.THT_CATALOG_DB_HOST;
|
||||
const database = process.env.THT_CATALOG_DB_NAME;
|
||||
const user = process.env.THT_CATALOG_MIGRATOR_USER;
|
||||
const passwordFile = process.env.THT_CATALOG_MIGRATOR_PASSWORD_FILE;
|
||||
if (!connectionString && (!host || !database || !user || !passwordFile)) {
|
||||
throw new Error("catalog migrator database configuration is required");
|
||||
}
|
||||
const pool = new Pool(connectionString ? { connectionString, max: 1 } : {
|
||||
host,
|
||||
port: Number(process.env.THT_CATALOG_DB_PORT ?? 5432),
|
||||
database,
|
||||
user,
|
||||
password: async () => (await readFile(passwordFile!, "utf8")).trim(),
|
||||
max: 1,
|
||||
});
|
||||
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
const provider: MigrationProvider = {
|
||||
async getMigrations() {
|
||||
return {
|
||||
"001_workspace_databases": initialMigration,
|
||||
"002_catalog_tables": catalogTablesMigration,
|
||||
"003_catalog_schema_sync": catalogSchemaSyncMigration,
|
||||
"004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration,
|
||||
"005_description_generation_runs": descriptionGenerationRunsMigration,
|
||||
"006_sensitive_data_flag": sensitiveDataFlagMigration,
|
||||
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
|
||||
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
|
||||
"009_ai_token_usage": aiTokenUsageMigration,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await new Migrator({ db, provider }).migrateToLatest();
|
||||
for (const item of result.results ?? []) {
|
||||
process.stdout.write(`${item.migrationName}: ${item.status}\n`);
|
||||
}
|
||||
if (result.error) throw result.error;
|
||||
} finally {
|
||||
await db.destroy();
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("workspace_databases")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("workspace_id", "text", (column) => column.notNull().unique())
|
||||
.addColumn("engine", "text", (column) => column.notNull())
|
||||
.addColumn("database_name", "text", (column) => column.notNull())
|
||||
.addColumn("schema_name", "text", (column) => column.notNull())
|
||||
.addColumn("version", "integer", (column) => column.notNull().defaultTo(1))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addCheckConstraint("workspace_databases_engine_check", sql`engine = 'postgres'`)
|
||||
.addCheckConstraint("workspace_databases_version_check", sql`version > 0`)
|
||||
.execute();
|
||||
|
||||
await db.schema.createTable("database_bindings")
|
||||
.addColumn("database_id", "uuid", (column) => column.primaryKey()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("transport", "text", (column) => column.notNull())
|
||||
.addColumn("host", "text")
|
||||
.addColumn("port", "integer")
|
||||
.addColumn("username", "text")
|
||||
.addColumn("base_url", "text")
|
||||
.addColumn("rest_path", "text")
|
||||
.addColumn("rest_auth", "text")
|
||||
.addColumn("tls_servername", "text")
|
||||
.addColumn("ssh_host", "text")
|
||||
.addColumn("ssh_port", "integer")
|
||||
.addColumn("ssh_username", "text")
|
||||
.addColumn("ssh_target_host", "text")
|
||||
.addColumn("ssh_target_port", "integer")
|
||||
.addColumn("connection_status", "text", (column) => column.notNull().defaultTo("untested"))
|
||||
.addColumn("tested_version", "integer")
|
||||
.addColumn("last_tested_at", "timestamptz")
|
||||
.addColumn("last_error_code", "text")
|
||||
.addColumn("last_error_message", "text")
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addCheckConstraint("database_bindings_transport_check", sql`transport in ('postgres_direct', 'rest_api', 'ssh_tunnel')`)
|
||||
.addCheckConstraint("database_bindings_status_check", sql`connection_status in ('untested', 'reachable', 'failed')`)
|
||||
.addCheckConstraint("database_bindings_port_check", sql`port is null or port between 1 and 65535`)
|
||||
.addCheckConstraint("database_bindings_ssh_port_check", sql`ssh_port is null or ssh_port between 1 and 65535`)
|
||||
.addCheckConstraint("database_bindings_ssh_target_port_check", sql`ssh_target_port is null or ssh_target_port between 1 and 65535`)
|
||||
.addCheckConstraint("database_bindings_transport_fields_check", sql`
|
||||
(transport = 'postgres_direct' and host is not null and port is not null and username is not null)
|
||||
or (transport = 'rest_api' and base_url is not null and rest_path is not null and rest_auth is not null)
|
||||
or (transport = 'ssh_tunnel' and username is not null and ssh_host is not null
|
||||
and ssh_port is not null and ssh_username is not null and ssh_target_host is not null
|
||||
and ssh_target_port is not null)
|
||||
`)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("database_bindings").execute();
|
||||
await db.schema.dropTable("workspace_databases").execute();
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("catalog_tables")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("name", "text", (column) => column.notNull())
|
||||
.addColumn("source_comment", "text")
|
||||
.addColumn("description", "text")
|
||||
.addColumn("generated_description", "text")
|
||||
.addColumn("version", "integer", (column) => column.notNull().defaultTo(1))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_tables_database_name_key", ["database_id", "name"])
|
||||
.addCheckConstraint("catalog_tables_name_check", sql`char_length(name) between 1 and 128`)
|
||||
.addCheckConstraint("catalog_tables_version_check", sql`version > 0`)
|
||||
.execute();
|
||||
|
||||
await db.schema.createIndex("catalog_tables_database_id_idx")
|
||||
.on("catalog_tables").column("database_id").execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("catalog_tables").execute();
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.alterTable("workspace_databases")
|
||||
.addColumn("schema_synced_version", "integer")
|
||||
.addColumn("schema_synced_at", "timestamptz")
|
||||
.execute();
|
||||
|
||||
await db.schema.alterTable("catalog_tables")
|
||||
.addColumn("last_synced_database_version", "integer")
|
||||
.addColumn("last_synced_at", "timestamptz")
|
||||
.execute();
|
||||
|
||||
await db.schema.createTable("catalog_columns")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("table_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_tables.id").onDelete("cascade"))
|
||||
.addColumn("name", "text", (column) => column.notNull())
|
||||
.addColumn("ordinal_position", "integer", (column) => column.notNull())
|
||||
.addColumn("data_type", "text", (column) => column.notNull())
|
||||
.addColumn("is_nullable", "boolean", (column) => column.notNull())
|
||||
.addColumn("default_expression", "text")
|
||||
.addColumn("primary_key_position", "integer")
|
||||
.addColumn("source_comment", "text")
|
||||
.addColumn("description", "text")
|
||||
.addColumn("generated_description", "text")
|
||||
.addColumn("last_synced_database_version", "integer")
|
||||
.addColumn("last_synced_at", "timestamptz")
|
||||
.addColumn("version", "integer", (column) => column.notNull().defaultTo(1))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_columns_table_name_key", ["table_id", "name"])
|
||||
.addCheckConstraint("catalog_columns_name_check", sql`char_length(name) between 1 and 128`)
|
||||
.addCheckConstraint("catalog_columns_ordinal_check", sql`ordinal_position > 0`)
|
||||
.addCheckConstraint("catalog_columns_pk_position_check", sql`primary_key_position is null or primary_key_position > 0`)
|
||||
.addCheckConstraint("catalog_columns_version_check", sql`version > 0`)
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_columns_table_id_idx")
|
||||
.on("catalog_columns").column("table_id").execute();
|
||||
|
||||
await db.schema.createTable("catalog_relationships")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("constraint_name", "text", (column) => column.notNull())
|
||||
.addColumn("source_table_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_tables.id").onDelete("cascade"))
|
||||
.addColumn("target_table_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_tables.id").onDelete("cascade"))
|
||||
.addColumn("update_rule", "text", (column) => column.notNull())
|
||||
.addColumn("delete_rule", "text", (column) => column.notNull())
|
||||
.addColumn("deferrable", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.addColumn("initially_deferred", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.addColumn("last_synced_database_version", "integer")
|
||||
.addColumn("last_synced_at", "timestamptz")
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_relationships_source_constraint_key", ["source_table_id", "constraint_name"])
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_relationships_database_id_idx")
|
||||
.on("catalog_relationships").column("database_id").execute();
|
||||
|
||||
await db.schema.createTable("catalog_relationship_columns")
|
||||
.addColumn("relationship_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_relationships.id").onDelete("cascade"))
|
||||
.addColumn("position", "integer", (column) => column.notNull())
|
||||
.addColumn("source_column_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_columns.id").onDelete("cascade"))
|
||||
.addColumn("target_column_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_columns.id").onDelete("cascade"))
|
||||
.addPrimaryKeyConstraint("catalog_relationship_columns_pkey", ["relationship_id", "position"])
|
||||
.addCheckConstraint("catalog_relationship_columns_position_check", sql`position > 0`)
|
||||
.execute();
|
||||
|
||||
await db.schema.createTable("catalog_sync_runs")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("scope", "text", (column) => column.notNull())
|
||||
.addColumn("table_ids", "jsonb", (column) => column.notNull().defaultTo(sql`'[]'::jsonb`))
|
||||
.addColumn("state", "text", (column) => column.notNull())
|
||||
.addColumn("phase", "text", (column) => column.notNull())
|
||||
.addColumn("requested_database_version", "integer", (column) => column.notNull())
|
||||
.addColumn("observed_snapshot", "jsonb")
|
||||
.addColumn("planned_diff", "jsonb")
|
||||
.addColumn("confirmation_token", "text")
|
||||
.addColumn("counts", "jsonb", (column) => column.notNull().defaultTo(sql`'{}'::jsonb`))
|
||||
.addColumn("error_code", "text")
|
||||
.addColumn("error_message", "text")
|
||||
.addColumn("cancel_requested", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("started_at", "timestamptz")
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("finished_at", "timestamptz")
|
||||
.addColumn("heartbeat_at", "timestamptz")
|
||||
.addColumn("lease_owner", "text")
|
||||
.addColumn("lease_expires_at", "timestamptz")
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_sync_runs_database_created_idx")
|
||||
.on("catalog_sync_runs").columns(["database_id", "created_at"]).execute();
|
||||
await sql`CREATE UNIQUE INDEX catalog_sync_runs_one_active_per_database
|
||||
ON catalog_sync_runs (database_id)
|
||||
WHERE state IN ('queued', 'running', 'awaiting_confirmation', 'applying')`.execute(db);
|
||||
|
||||
await db.schema.createTable("catalog_sync_events")
|
||||
.addColumn("id", "bigserial", (column) => column.primaryKey())
|
||||
.addColumn("run_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_sync_runs.id").onDelete("cascade"))
|
||||
.addColumn("sequence", "integer", (column) => column.notNull())
|
||||
.addColumn("level", "text", (column) => column.notNull())
|
||||
.addColumn("event_type", "text", (column) => column.notNull())
|
||||
.addColumn("message", "text", (column) => column.notNull())
|
||||
.addColumn("data", "jsonb", (column) => column.notNull().defaultTo(sql`'{}'::jsonb`))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_sync_events_run_sequence_key", ["run_id", "sequence"])
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_sync_events_run_id_idx")
|
||||
.on("catalog_sync_events").columns(["run_id", "sequence"]).execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("catalog_sync_events").execute();
|
||||
await db.schema.dropTable("catalog_sync_runs").execute();
|
||||
await db.schema.dropTable("catalog_relationship_columns").execute();
|
||||
await db.schema.dropTable("catalog_relationships").execute();
|
||||
await db.schema.dropTable("catalog_columns").execute();
|
||||
await db.schema.alterTable("catalog_tables")
|
||||
.dropColumn("last_synced_database_version")
|
||||
.dropColumn("last_synced_at")
|
||||
.execute();
|
||||
await db.schema.alterTable("workspace_databases")
|
||||
.dropColumn("schema_synced_version")
|
||||
.dropColumn("schema_synced_at")
|
||||
.execute();
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
/**
|
||||
* `catalog_sync_events.id` is the first catalog-owned identity sequence.
|
||||
* Table default privileges do not cover sequences, and without USAGE the
|
||||
* runtime can create a run but cannot append its first event.
|
||||
*/
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await sql`DO $catalog_privileges$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'thothii_catalog_runtime') THEN
|
||||
EXECUTE 'GRANT USAGE, SELECT ON SEQUENCE catalog_sync_events_id_seq TO thothii_catalog_runtime';
|
||||
EXECUTE 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO thothii_catalog_runtime';
|
||||
END IF;
|
||||
END
|
||||
$catalog_privileges$`.execute(db);
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await sql`DO $catalog_privileges$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'thothii_catalog_runtime') THEN
|
||||
EXECUTE 'ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE USAGE, SELECT ON SEQUENCES FROM thothii_catalog_runtime';
|
||||
EXECUTE 'REVOKE USAGE, SELECT ON SEQUENCE catalog_sync_events_id_seq FROM thothii_catalog_runtime';
|
||||
END IF;
|
||||
END
|
||||
$catalog_privileges$`.execute(db);
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("description_generation_runs")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("scope", "text", (column) => column.notNull())
|
||||
.addColumn("model_id", "text", (column) => column.notNull())
|
||||
.addColumn("language", "text", (column) => column.notNull())
|
||||
.addColumn("status", "text", (column) => column.notNull())
|
||||
.addColumn("total", "integer", (column) => column.notNull())
|
||||
.addColumn("processed", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("generated", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("non_generatable", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("failed", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("started_at", "timestamptz")
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("finished_at", "timestamptz")
|
||||
.addColumn("error_summary", "text")
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_scope_check",
|
||||
sql`scope in ('selected_columns', 'selected_tables', 'all', 'missing')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_model_id_check",
|
||||
sql`model_id ~ '^[a-z][a-z0-9._-]{0,63}$'`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_language_check",
|
||||
sql`language in ('en', 'it')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_status_check",
|
||||
sql`status in (
|
||||
'queued', 'running', 'completed', 'completed_with_errors',
|
||||
'cancelled', 'failed', 'interrupted'
|
||||
)`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_counters_check",
|
||||
sql`total > 0
|
||||
and processed between 0 and total
|
||||
and generated >= 0
|
||||
and non_generatable >= 0
|
||||
and failed >= 0
|
||||
and generated + non_generatable + failed <= processed`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_error_summary_check",
|
||||
sql`error_summary is null or char_length(error_summary) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
await db.schema.createIndex("description_generation_runs_database_created_idx")
|
||||
.on("description_generation_runs")
|
||||
.columns(["database_id", "created_at"])
|
||||
.execute();
|
||||
await sql`CREATE UNIQUE INDEX description_generation_runs_one_active
|
||||
ON description_generation_runs ((true))
|
||||
WHERE status IN ('queued', 'running')`.execute(db);
|
||||
|
||||
await db.schema.createTable("description_generation_events")
|
||||
.addColumn("run_id", "uuid", (column) => column.notNull()
|
||||
.references("description_generation_runs.id").onDelete("cascade"))
|
||||
.addColumn("sequence", "integer", (column) => column.notNull())
|
||||
.addColumn("level", "text", (column) => column.notNull())
|
||||
.addColumn("message", "text", (column) => column.notNull())
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addPrimaryKeyConstraint("description_generation_events_pkey", ["run_id", "sequence"])
|
||||
.addCheckConstraint("description_generation_events_sequence_check", sql`sequence > 0`)
|
||||
.addCheckConstraint(
|
||||
"description_generation_events_level_check",
|
||||
sql`level in ('info', 'warning', 'error')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_events_message_check",
|
||||
sql`char_length(message) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("description_generation_events").execute();
|
||||
await db.schema.dropTable("description_generation_runs").execute();
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.alterTable("catalog_columns")
|
||||
.addColumn("sensitive", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.alterTable("catalog_columns").dropColumn("sensitive").execute();
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("sensitive_data_suggestion_runs")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("scope", "text", (column) => column.notNull())
|
||||
.addColumn("model_id", "text", (column) => column.notNull())
|
||||
.addColumn("status", "text", (column) => column.notNull())
|
||||
.addColumn("total", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("suggested_sensitive", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("suggested_non_sensitive", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("started_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("finished_at", "timestamptz")
|
||||
.addColumn("error_summary", "text")
|
||||
.addCheckConstraint(
|
||||
"sensitive_data_suggestion_runs_scope_check",
|
||||
sql`scope in ('selected_columns', 'selected_tables', 'all')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"sensitive_data_suggestion_runs_model_id_check",
|
||||
sql`model_id ~ '^[a-z][a-z0-9._-]{0,63}$'`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"sensitive_data_suggestion_runs_status_check",
|
||||
sql`status in ('running', 'completed', 'failed', 'interrupted')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"sensitive_data_suggestion_runs_counters_check",
|
||||
sql`total >= 0
|
||||
and suggested_sensitive >= 0
|
||||
and suggested_non_sensitive >= 0
|
||||
and suggested_sensitive + suggested_non_sensitive <= total`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"sensitive_data_suggestion_runs_error_summary_check",
|
||||
sql`error_summary is null or char_length(error_summary) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
|
||||
await db.schema.createIndex("sensitive_data_suggestion_runs_database_created_idx")
|
||||
.on("sensitive_data_suggestion_runs")
|
||||
.columns(["database_id", "created_at"])
|
||||
.execute();
|
||||
|
||||
await db.schema.createTable("sensitive_data_suggestion_events")
|
||||
.addColumn("run_id", "uuid", (column) => column.notNull()
|
||||
.references("sensitive_data_suggestion_runs.id").onDelete("cascade"))
|
||||
.addColumn("sequence", "integer", (column) => column.notNull())
|
||||
.addColumn("level", "text", (column) => column.notNull())
|
||||
.addColumn("message", "text", (column) => column.notNull())
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addPrimaryKeyConstraint("sensitive_data_suggestion_events_pkey", ["run_id", "sequence"])
|
||||
.addCheckConstraint("sensitive_data_suggestion_events_sequence_check", sql`sequence > 0`)
|
||||
.addCheckConstraint(
|
||||
"sensitive_data_suggestion_events_level_check",
|
||||
sql`level in ('info', 'warning', 'error')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"sensitive_data_suggestion_events_message_check",
|
||||
sql`char_length(message) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("sensitive_data_suggestion_events").execute();
|
||||
await db.schema.dropTable("sensitive_data_suggestion_runs").execute();
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("catalog_logical_relationships")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("source_column_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_columns.id").onDelete("cascade"))
|
||||
.addColumn("target_column_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_columns.id").onDelete("cascade"))
|
||||
.addColumn("generated", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.addColumn("deleted_at", "timestamptz")
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint(
|
||||
"catalog_logical_relationships_endpoint_key",
|
||||
["database_id", "source_column_id", "target_column_id"],
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"catalog_logical_relationships_distinct_columns_check",
|
||||
sql`source_column_id <> target_column_id`,
|
||||
)
|
||||
.execute();
|
||||
|
||||
await db.schema.createIndex("catalog_logical_relationships_database_deleted_idx")
|
||||
.on("catalog_logical_relationships")
|
||||
.columns(["database_id", "deleted_at"])
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("catalog_logical_relationships").execute();
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
for (const table of ["description_generation_runs", "sensitive_data_suggestion_runs"] as const) {
|
||||
await db.schema.alterTable(table)
|
||||
.addColumn("input_tokens", "integer", (col) => col.notNull().defaultTo(0))
|
||||
.addColumn("cache_read_tokens", "integer", (col) => col.notNull().defaultTo(0))
|
||||
.addColumn("output_tokens", "integer", (col) => col.notNull().defaultTo(0))
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
for (const table of ["sensitive_data_suggestion_runs", "description_generation_runs"] as const) {
|
||||
await db.schema.alterTable(table)
|
||||
.dropColumn("input_tokens").dropColumn("cache_read_tokens").dropColumn("output_tokens")
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { z } from "zod";
|
||||
import type { ResolvedMetadataGenerationModel } from "./metadata-generation-models.js";
|
||||
|
||||
const MAX_HELPER_OUTPUT_BYTES = 64 * 1024;
|
||||
const helperOutputSchema = z.discriminatedUnion("ok", [
|
||||
z.object({ ok: z.literal(true), content: z.string(), usage: z.object({ input: z.number().int().nonnegative(), cacheRead: z.number().int().nonnegative(), output: z.number().int().nonnegative() }).strict().optional() }).strict(),
|
||||
z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(),
|
||||
]);
|
||||
|
||||
export interface ModelCompletionMessage {
|
||||
role: "system" | "user";
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ModelCompletionRequest {
|
||||
model: ResolvedMetadataGenerationModel;
|
||||
messages: readonly ModelCompletionMessage[];
|
||||
signal: AbortSignal;
|
||||
}
|
||||
export interface ModelCompletionUsage { input: number; cacheRead: number; output: number; }
|
||||
export interface ModelCompletionResult { content: string; usage: ModelCompletionUsage; }
|
||||
|
||||
/** The provider boundary used by Description Generation. */
|
||||
export interface ModelCompleter {
|
||||
complete(request: ModelCompletionRequest): Promise<string | ModelCompletionResult>;
|
||||
}
|
||||
|
||||
export class ModelCompletionProviderError extends Error {
|
||||
constructor() {
|
||||
super("model completion failed");
|
||||
this.name = "ModelCompletionProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ModelCompletionCancelledError extends Error {
|
||||
constructor() {
|
||||
super("model completion cancelled");
|
||||
this.name = "ModelCompletionCancelledError";
|
||||
}
|
||||
}
|
||||
|
||||
export class PythonModelCompleter implements ModelCompleter {
|
||||
constructor(private readonly options: {
|
||||
pythonExecutable: string;
|
||||
cwd: string;
|
||||
helperModule?: string;
|
||||
timeoutMs?: number;
|
||||
terminationGraceMs?: number;
|
||||
}) {}
|
||||
|
||||
async complete(request: ModelCompletionRequest): Promise<ModelCompletionResult> {
|
||||
if (request.signal.aborted) throw new ModelCompletionCancelledError();
|
||||
const payload = {
|
||||
model: `${request.model.provider}/${request.model.model}`,
|
||||
...(request.model.apiKey === undefined ? {} : { api_key: request.model.apiKey }),
|
||||
messages: request.messages.map((message) => ({ ...message })),
|
||||
...(request.model.endpoint?.baseUrl === undefined
|
||||
? {}
|
||||
: { api_base: request.model.endpoint.baseUrl }),
|
||||
...(request.model.endpoint?.apiVersion === undefined
|
||||
? {}
|
||||
: { api_version: request.model.endpoint.apiVersion }),
|
||||
...(request.model.disableThinking === true ? { disable_thinking: true } : {}),
|
||||
};
|
||||
|
||||
return await new Promise<ModelCompletionResult>((resolve, reject) => {
|
||||
const child = spawn(
|
||||
this.options.pythonExecutable,
|
||||
["-m", this.options.helperModule ?? "tht.internal.litellm_completion"],
|
||||
{
|
||||
cwd: this.options.cwd,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
let stdout = "";
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let killFallback: ReturnType<typeof setTimeout> | undefined;
|
||||
let terminatingWith: Error | undefined;
|
||||
const cleanup = () => {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
if (killFallback) clearTimeout(killFallback);
|
||||
request.signal.removeEventListener("abort", cancel);
|
||||
};
|
||||
const fail = (error: Error = new ModelCompletionProviderError()) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const terminate = (error: Error) => {
|
||||
if (settled || terminatingWith) return;
|
||||
terminatingWith = error;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
fail(error);
|
||||
return;
|
||||
}
|
||||
killFallback = setTimeout(() => {
|
||||
if (settled || child.exitCode !== null || child.signalCode !== null) return;
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
fail(error);
|
||||
}
|
||||
}, this.options.terminationGraceMs ?? 250);
|
||||
};
|
||||
const cancel = () => {
|
||||
terminate(new ModelCompletionCancelledError());
|
||||
};
|
||||
timeout = setTimeout(() => {
|
||||
terminate(new ModelCompletionProviderError());
|
||||
}, this.options.timeoutMs ?? 120_000);
|
||||
request.signal.addEventListener("abort", cancel, { once: true });
|
||||
if (request.signal.aborted) cancel();
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (terminatingWith) return;
|
||||
stdout += chunk;
|
||||
if (Buffer.byteLength(stdout, "utf8") > MAX_HELPER_OUTPUT_BYTES) {
|
||||
terminate(new ModelCompletionProviderError());
|
||||
}
|
||||
});
|
||||
// Helper and provider diagnostics are deliberately not copied into application logs.
|
||||
child.stderr.resume();
|
||||
child.once("error", () => fail(terminatingWith ?? new ModelCompletionProviderError()));
|
||||
child.once("close", (code) => {
|
||||
if (settled) return;
|
||||
if (terminatingWith) return fail(terminatingWith);
|
||||
try {
|
||||
if (code !== 0) return fail();
|
||||
const output = helperOutputSchema.parse(JSON.parse(stdout));
|
||||
if (!output.ok) return fail();
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve({ content: output.content, usage: output.usage ?? { input: 0, cacheRead: 0, output: 0 } });
|
||||
} catch {
|
||||
fail();
|
||||
}
|
||||
});
|
||||
child.stdin.once("error", () => {
|
||||
if (terminatingWith) return;
|
||||
terminate(new ModelCompletionProviderError());
|
||||
});
|
||||
child.stdin.end(JSON.stringify(payload));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
import { CatalogOperationInProgressError } from "./types.js";
|
||||
|
||||
/** Serializes mutable catalog operations for each Workspace Database. */
|
||||
export class CatalogOperationCoordinator {
|
||||
private readonly active = new Map<
|
||||
string,
|
||||
{ token: symbol; owner: "catalog_operation" | "description_generation" }
|
||||
>();
|
||||
|
||||
reserve(
|
||||
databaseId: string,
|
||||
owner: "catalog_operation" | "description_generation" = "catalog_operation",
|
||||
): () => void {
|
||||
if (this.active.has(databaseId)) {
|
||||
throw new CatalogOperationInProgressError("A database operation is already in progress");
|
||||
}
|
||||
const token = Symbol(databaseId);
|
||||
this.active.set(databaseId, { token, owner });
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
if (this.active.get(databaseId)?.token === token) this.active.delete(databaseId);
|
||||
};
|
||||
}
|
||||
|
||||
/** Administrative recovery for a reservation whose owning local operation is no longer live. */
|
||||
releaseStale(databaseId: string, owner: "description_generation"): void {
|
||||
if (this.active.get(databaseId)?.owner === owner) this.active.delete(databaseId);
|
||||
}
|
||||
|
||||
async run<T>(databaseId: string, operation: () => Promise<T>): Promise<T> {
|
||||
const release = this.reserve(databaseId);
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
async runMany<T>(databaseIds: readonly string[], operation: () => Promise<T>): Promise<T> {
|
||||
const releases: Array<() => void> = [];
|
||||
try {
|
||||
for (const databaseId of [...new Set(databaseIds)].sort()) {
|
||||
releases.push(this.reserve(databaseId));
|
||||
}
|
||||
return await operation();
|
||||
} finally {
|
||||
for (const release of releases.reverse()) release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,262 +0,0 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Duplex } from "node:stream";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { Client, type ClientConfig } from "pg";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||
import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js";
|
||||
|
||||
export interface CatalogDatabaseClient {
|
||||
query(sql: string, values: readonly unknown[]): Promise<{ rows: Array<Record<string, unknown>> }>;
|
||||
end(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface CatalogPostgresAccess {
|
||||
connect(database: WorkspaceDatabase, signal: AbortSignal): Promise<CatalogDatabaseClient>;
|
||||
}
|
||||
|
||||
type SpawnSsh = (
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
options: { env: NodeJS.ProcessEnv },
|
||||
) => ChildProcessWithoutNullStreams;
|
||||
|
||||
interface AccessDependencies {
|
||||
createClient?: (config: ClientConfig) => Client;
|
||||
spawnSsh?: SpawnSsh;
|
||||
sshBinary?: string;
|
||||
askpassPath?: string;
|
||||
connectTimeoutMs?: number;
|
||||
}
|
||||
|
||||
function required(value: string | number | undefined): string | number {
|
||||
if (value === undefined || value === "") throw new CatalogConnectorError("Database binding is incomplete");
|
||||
return value;
|
||||
}
|
||||
|
||||
function connectionSsl(ca: string | undefined, servername: string | undefined): ClientConfig["ssl"] {
|
||||
if (!ca && !servername) return false;
|
||||
return {
|
||||
...(ca ? { ca } : {}),
|
||||
...(servername ? { servername } : {}),
|
||||
rejectUnauthorized: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildSshArguments(input: {
|
||||
sshHost: string;
|
||||
sshPort: number;
|
||||
sshUsername: string;
|
||||
targetHost: string;
|
||||
targetPort: number;
|
||||
privateKeyFile: string;
|
||||
knownHostsFile: string;
|
||||
passphraseFile?: string;
|
||||
connectTimeoutMs: number;
|
||||
}): string[] {
|
||||
const batchMode = input.passphraseFile ? "no" : "yes";
|
||||
return [
|
||||
"-F", "/dev/null",
|
||||
"-T",
|
||||
"-o", `BatchMode=${batchMode}`,
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", `UserKnownHostsFile=${input.knownHostsFile}`,
|
||||
"-o", "GlobalKnownHostsFile=/dev/null",
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", `IdentityFile=${input.privateKeyFile}`,
|
||||
"-o", "PreferredAuthentications=publickey",
|
||||
"-o", "PasswordAuthentication=no",
|
||||
"-o", "KbdInteractiveAuthentication=no",
|
||||
"-o", "ConnectionAttempts=1",
|
||||
"-o", `ConnectTimeout=${Math.max(1, Math.ceil(input.connectTimeoutMs / 1_000))}`,
|
||||
"-o", "ServerAliveInterval=5",
|
||||
"-o", "ServerAliveCountMax=1",
|
||||
"-o", "NumberOfPasswordPrompts=1",
|
||||
"-o", "RequestTTY=no",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-p", String(input.sshPort),
|
||||
"-W", `${input.targetHost}:${input.targetPort}`,
|
||||
"--", `${input.sshUsername}@${input.sshHost}`,
|
||||
];
|
||||
}
|
||||
|
||||
async function stopChild(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
child.kill("SIGTERM");
|
||||
await Promise.race([
|
||||
new Promise<void>((resolve) => child.once("exit", () => resolve())),
|
||||
delay(500).then(() => undefined),
|
||||
]);
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
}
|
||||
|
||||
function sshDuplex(child: ChildProcessWithoutNullStreams): Duplex {
|
||||
let ended = false;
|
||||
let stream: Duplex;
|
||||
const forward = () => {
|
||||
let chunk: Buffer | string | null;
|
||||
while ((chunk = child.stdout.read() as Buffer | string | null) !== null) {
|
||||
if (!stream.push(chunk)) break;
|
||||
}
|
||||
};
|
||||
const finish = () => {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
stream.push(null);
|
||||
};
|
||||
const fail = (error: Error) => stream.destroy(error);
|
||||
stream = new Duplex({
|
||||
read: forward,
|
||||
write: (chunk, encoding, callback) => child.stdin.write(chunk, encoding, callback),
|
||||
final: (callback) => child.stdin.end(callback),
|
||||
destroy: (error, callback) => {
|
||||
child.stdout.off("readable", forward);
|
||||
child.stdout.off("end", finish);
|
||||
child.stdout.off("error", fail);
|
||||
child.stdin.off("error", fail);
|
||||
callback(error);
|
||||
},
|
||||
});
|
||||
child.stdout.on("readable", forward);
|
||||
child.stdout.once("end", finish);
|
||||
child.stdout.once("error", fail);
|
||||
child.stdin.once("error", fail);
|
||||
return stream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep connection module for direct and SSH-forwarded PostgreSQL access. It owns secret leases,
|
||||
* TLS, OpenSSH lifecycle, abort propagation, and pg cleanup behind one connect interface.
|
||||
*/
|
||||
export class ConcreteCatalogPostgresAccess implements CatalogPostgresAccess {
|
||||
private readonly createClient: (config: ClientConfig) => Client;
|
||||
private readonly spawnSsh: SpawnSsh;
|
||||
private readonly sshBinary: string;
|
||||
private readonly askpassPath: string;
|
||||
private readonly connectTimeoutMs: number;
|
||||
|
||||
constructor(
|
||||
private readonly secretStore: WorkspaceSecretStore,
|
||||
dependencies: AccessDependencies = {},
|
||||
) {
|
||||
this.createClient = dependencies.createClient ?? ((config) => new Client(config));
|
||||
this.spawnSsh = dependencies.spawnSsh ?? ((command, args, options) => (
|
||||
spawn(command, [...args], { ...options, stdio: ["pipe", "pipe", "pipe"] })
|
||||
));
|
||||
this.sshBinary = dependencies.sshBinary ?? process.env.THT_SSH_BIN ?? "ssh";
|
||||
this.askpassPath = dependencies.askpassPath
|
||||
?? process.env.THT_SSH_ASKPASS_BIN
|
||||
?? fileURLToPath(new URL("../../scripts/ssh-askpass.mjs", import.meta.url));
|
||||
this.connectTimeoutMs = dependencies.connectTimeoutMs ?? 5_000;
|
||||
}
|
||||
|
||||
async connect(database: WorkspaceDatabase, signal: AbortSignal): Promise<CatalogDatabaseClient> {
|
||||
if (database.binding.transport === "rest_api") {
|
||||
throw new CatalogConnectorError("REST is not a PostgreSQL wire binding");
|
||||
}
|
||||
const ids: string[] = [CATALOG_SECRET_IDS.password, CATALOG_SECRET_IDS.tlsCa];
|
||||
if (database.binding.transport === "ssh_tunnel") {
|
||||
ids.push(
|
||||
CATALOG_SECRET_IDS.sshPrivateKey,
|
||||
CATALOG_SECRET_IDS.sshPrivateKeyPassphrase,
|
||||
CATALOG_SECRET_IDS.sshKnownHosts,
|
||||
);
|
||||
}
|
||||
const materialized = this.secretStore.materialize(database.workspaceId, ids);
|
||||
let child: ChildProcessWithoutNullStreams | undefined;
|
||||
let stream: Duplex | undefined;
|
||||
let client: Client | undefined;
|
||||
let ended = false;
|
||||
const close = async () => {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
if (client) await client.end().catch(() => undefined);
|
||||
stream?.destroy();
|
||||
if (child) await stopChild(child);
|
||||
materialized.release();
|
||||
};
|
||||
const abort = () => { void close(); };
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
|
||||
try {
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
const passwordFile = materialized.files.get(CATALOG_SECRET_IDS.password);
|
||||
if (!passwordFile) throw new CatalogConnectorError("Database password is not configured");
|
||||
const password = await readFile(passwordFile, "utf8");
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
const tlsCaFile = materialized.files.get(CATALOG_SECRET_IDS.tlsCa);
|
||||
const tlsCa = tlsCaFile ? await readFile(tlsCaFile, "utf8") : undefined;
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
let host: string;
|
||||
let port: number;
|
||||
|
||||
if (database.binding.transport === "ssh_tunnel") {
|
||||
const privateKeyFile = materialized.files.get(CATALOG_SECRET_IDS.sshPrivateKey);
|
||||
const knownHostsFile = materialized.files.get(CATALOG_SECRET_IDS.sshKnownHosts);
|
||||
if (!privateKeyFile || !knownHostsFile) {
|
||||
throw new CatalogConnectorError("SSH private key and known hosts are required");
|
||||
}
|
||||
const passphraseFile = materialized.files.get(CATALOG_SECRET_IDS.sshPrivateKeyPassphrase);
|
||||
host = String(required(database.binding.sshTargetHost));
|
||||
port = Number(required(database.binding.sshTargetPort));
|
||||
const args = buildSshArguments({
|
||||
sshHost: String(required(database.binding.sshHost)),
|
||||
sshPort: Number(required(database.binding.sshPort)),
|
||||
sshUsername: String(required(database.binding.sshUsername)),
|
||||
targetHost: host,
|
||||
targetPort: port,
|
||||
privateKeyFile,
|
||||
knownHostsFile,
|
||||
passphraseFile,
|
||||
connectTimeoutMs: this.connectTimeoutMs,
|
||||
});
|
||||
child = this.spawnSsh(this.sshBinary, args, {
|
||||
env: {
|
||||
...process.env,
|
||||
LC_ALL: "C",
|
||||
...(passphraseFile ? {
|
||||
DISPLAY: "thothii",
|
||||
SSH_ASKPASS: this.askpassPath,
|
||||
SSH_ASKPASS_REQUIRE: "force",
|
||||
THT_SSH_PASSPHRASE_FILE: passphraseFile,
|
||||
} : {}),
|
||||
},
|
||||
});
|
||||
stream = sshDuplex(child);
|
||||
child.once("error", () => stream?.destroy(new CatalogConnectorError("SSH process failed")));
|
||||
child.once("exit", (code) => {
|
||||
if (!ended && code !== 0) stream?.destroy(new CatalogConnectorError("SSH tunnel failed"));
|
||||
});
|
||||
child.stderr.on("data", () => undefined);
|
||||
} else {
|
||||
host = String(required(database.binding.host));
|
||||
port = Number(required(database.binding.port));
|
||||
}
|
||||
|
||||
client = this.createClient({
|
||||
host,
|
||||
port,
|
||||
database: database.databaseName,
|
||||
user: String(required(database.binding.username)),
|
||||
password,
|
||||
ssl: connectionSsl(tlsCa, database.binding.tlsServername),
|
||||
connectionTimeoutMillis: this.connectTimeoutMs,
|
||||
...(stream ? { stream: () => stream } : {}),
|
||||
});
|
||||
await client.connect();
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
return {
|
||||
query: async (sql, values) => await client!.query(sql, [...values]),
|
||||
end: close,
|
||||
};
|
||||
} catch (error) {
|
||||
await close();
|
||||
if (error instanceof CatalogConnectorError) throw error;
|
||||
throw new CatalogConnectorError("PostgreSQL connector failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,498 +0,0 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { z } from "zod";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||
import {
|
||||
CatalogConnectorError,
|
||||
CatalogSchemaCapabilityUnavailableError,
|
||||
type CatalogSyncPhase,
|
||||
type ObservedCatalogColumn,
|
||||
type ObservedCatalogRelationship,
|
||||
type ObservedCatalogTable,
|
||||
type ObservedSchemaSnapshot,
|
||||
type WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
|
||||
export type CatalogSchemaScanProgress = (
|
||||
phase: Extract<CatalogSyncPhase, "connecting" | "scanning_tables" | "scanning_columns" | "scanning_relationships">,
|
||||
counts?: { tables?: number; columns?: number; relationships?: number },
|
||||
) => Promise<void> | void;
|
||||
|
||||
export interface CatalogSchemaIntrospector {
|
||||
scan(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot>;
|
||||
}
|
||||
|
||||
const identifier = z.string().min(1).max(128);
|
||||
const nullableText = z.string().nullable();
|
||||
const capability = z.enum(["available", "unavailable"]);
|
||||
const restSnapshotSchema = z.object({
|
||||
schemaVersion: z.literal(1),
|
||||
capabilities: z.object({
|
||||
tables: capability,
|
||||
columns: capability,
|
||||
relationships: capability,
|
||||
}).strict(),
|
||||
tables: z.array(z.object({
|
||||
name: identifier,
|
||||
sourceComment: nullableText,
|
||||
}).strict()),
|
||||
columns: z.array(z.object({
|
||||
tableName: identifier,
|
||||
name: identifier,
|
||||
ordinalPosition: z.number().int().positive(),
|
||||
dataType: z.string().min(1).max(2_000),
|
||||
isNullable: z.boolean(),
|
||||
defaultExpression: nullableText,
|
||||
primaryKeyPosition: z.number().int().positive().nullable(),
|
||||
sourceComment: nullableText,
|
||||
}).strict()),
|
||||
relationships: z.array(z.object({
|
||||
constraintName: identifier,
|
||||
sourceTableName: identifier,
|
||||
targetTableName: identifier,
|
||||
updateRule: z.string().min(1).max(64),
|
||||
deleteRule: z.string().min(1).max(64),
|
||||
deferrable: z.boolean(),
|
||||
initiallyDeferred: z.boolean(),
|
||||
columns: z.array(z.object({
|
||||
position: z.number().int().positive(),
|
||||
sourceColumnName: identifier,
|
||||
targetColumnName: identifier,
|
||||
}).strict()).min(1),
|
||||
}).strict()),
|
||||
}).strict();
|
||||
|
||||
function sqlString(value: string): string {
|
||||
return `'${value.replaceAll("'", "''")}'`;
|
||||
}
|
||||
|
||||
function restSnapshotQuery(schemaName: string): string {
|
||||
const schema = sqlString(schemaName);
|
||||
return `WITH target_schema AS (
|
||||
SELECT oid
|
||||
FROM pg_catalog.pg_namespace
|
||||
WHERE nspname = ${schema}
|
||||
),
|
||||
observed_tables AS (
|
||||
SELECT c.oid,
|
||||
c.relname AS name,
|
||||
d.description AS source_comment
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN target_schema n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0
|
||||
WHERE c.relkind IN ('r', 'p')
|
||||
),
|
||||
primary_key_columns AS (
|
||||
SELECT i.indrelid AS table_oid,
|
||||
key.attnum,
|
||||
key.ordinality::integer AS position
|
||||
FROM pg_catalog.pg_index i
|
||||
CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS key(attnum, ordinality)
|
||||
WHERE i.indisprimary
|
||||
),
|
||||
observed_columns AS (
|
||||
SELECT table_info.name AS table_name,
|
||||
a.attname AS name,
|
||||
a.attnum::integer AS ordinal_position,
|
||||
pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||
NOT a.attnotnull AS is_nullable,
|
||||
pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
|
||||
pk.position AS primary_key_position,
|
||||
d.description AS source_comment
|
||||
FROM observed_tables table_info
|
||||
JOIN pg_catalog.pg_attribute a ON a.attrelid = table_info.oid
|
||||
LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid = table_info.oid AND ad.adnum = a.attnum
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = table_info.oid AND d.objsubid = a.attnum
|
||||
LEFT JOIN primary_key_columns pk ON pk.table_oid = table_info.oid AND pk.attnum = a.attnum
|
||||
WHERE a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
),
|
||||
relationship_pairs AS (
|
||||
SELECT con.oid AS constraint_oid,
|
||||
con.conname AS constraint_name,
|
||||
source_table.relname AS source_table_name,
|
||||
target_table.relname AS target_table_name,
|
||||
CASE con.confupdtype
|
||||
WHEN 'a' THEN 'NO ACTION'
|
||||
WHEN 'r' THEN 'RESTRICT'
|
||||
WHEN 'c' THEN 'CASCADE'
|
||||
WHEN 'n' THEN 'SET NULL'
|
||||
WHEN 'd' THEN 'SET DEFAULT'
|
||||
END AS update_rule,
|
||||
CASE con.confdeltype
|
||||
WHEN 'a' THEN 'NO ACTION'
|
||||
WHEN 'r' THEN 'RESTRICT'
|
||||
WHEN 'c' THEN 'CASCADE'
|
||||
WHEN 'n' THEN 'SET NULL'
|
||||
WHEN 'd' THEN 'SET DEFAULT'
|
||||
END AS delete_rule,
|
||||
con.condeferrable AS is_deferrable,
|
||||
con.condeferred AS initially_deferred,
|
||||
source_key.ordinality::integer AS position,
|
||||
source_column.attname AS source_column_name,
|
||||
target_column.attname AS target_column_name
|
||||
FROM pg_catalog.pg_constraint con
|
||||
JOIN pg_catalog.pg_class source_table ON source_table.oid = con.conrelid
|
||||
JOIN target_schema source_namespace ON source_namespace.oid = source_table.relnamespace
|
||||
JOIN pg_catalog.pg_class target_table ON target_table.oid = con.confrelid
|
||||
JOIN target_schema target_namespace ON target_namespace.oid = target_table.relnamespace
|
||||
JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) ON true
|
||||
JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS target_key(attnum, ordinality)
|
||||
ON target_key.ordinality = source_key.ordinality
|
||||
JOIN pg_catalog.pg_attribute source_column
|
||||
ON source_column.attrelid = source_table.oid AND source_column.attnum = source_key.attnum
|
||||
JOIN pg_catalog.pg_attribute target_column
|
||||
ON target_column.attrelid = target_table.oid AND target_column.attnum = target_key.attnum
|
||||
WHERE con.contype = 'f'
|
||||
),
|
||||
observed_relationships AS (
|
||||
SELECT constraint_oid,
|
||||
constraint_name,
|
||||
source_table_name,
|
||||
target_table_name,
|
||||
update_rule,
|
||||
delete_rule,
|
||||
is_deferrable,
|
||||
initially_deferred,
|
||||
pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object(
|
||||
'position', position,
|
||||
'sourceColumnName', source_column_name,
|
||||
'targetColumnName', target_column_name
|
||||
) ORDER BY position
|
||||
) AS columns
|
||||
FROM relationship_pairs
|
||||
GROUP BY constraint_oid, constraint_name, source_table_name, target_table_name,
|
||||
update_rule, delete_rule, is_deferrable, initially_deferred
|
||||
)
|
||||
SELECT 1 AS "schemaVersion",
|
||||
pg_catalog.jsonb_build_object(
|
||||
'tables', 'available',
|
||||
'columns', 'available',
|
||||
'relationships', 'available'
|
||||
) AS capabilities,
|
||||
COALESCE((
|
||||
SELECT pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object('name', name, 'sourceComment', source_comment)
|
||||
ORDER BY name
|
||||
)
|
||||
FROM observed_tables
|
||||
), '[]'::jsonb) AS tables,
|
||||
COALESCE((
|
||||
SELECT pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object(
|
||||
'tableName', table_name,
|
||||
'name', name,
|
||||
'ordinalPosition', ordinal_position,
|
||||
'dataType', data_type,
|
||||
'isNullable', is_nullable,
|
||||
'defaultExpression', default_expression,
|
||||
'primaryKeyPosition', primary_key_position,
|
||||
'sourceComment', source_comment
|
||||
) ORDER BY table_name, ordinal_position
|
||||
)
|
||||
FROM observed_columns
|
||||
), '[]'::jsonb) AS columns,
|
||||
COALESCE((
|
||||
SELECT pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object(
|
||||
'constraintName', constraint_name,
|
||||
'sourceTableName', source_table_name,
|
||||
'targetTableName', target_table_name,
|
||||
'updateRule', update_rule,
|
||||
'deleteRule', delete_rule,
|
||||
'deferrable', is_deferrable,
|
||||
'initiallyDeferred', initially_deferred,
|
||||
'columns', columns
|
||||
) ORDER BY source_table_name, constraint_name
|
||||
)
|
||||
FROM observed_relationships
|
||||
), '[]'::jsonb) AS relationships
|
||||
FROM target_schema`;
|
||||
}
|
||||
|
||||
function required(value: string | undefined): string {
|
||||
if (!value) throw new CatalogConnectorError("Database binding is incomplete");
|
||||
return value;
|
||||
}
|
||||
|
||||
function textOrNull(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function actionRule(value: unknown): string {
|
||||
const rules: Record<string, string> = {
|
||||
a: "NO ACTION",
|
||||
r: "RESTRICT",
|
||||
c: "CASCADE",
|
||||
n: "SET NULL",
|
||||
d: "SET DEFAULT",
|
||||
};
|
||||
const rule = rules[String(value)];
|
||||
if (!rule) throw new CatalogConnectorError("Schema introspection returned an unknown relationship action");
|
||||
return rule;
|
||||
}
|
||||
|
||||
function normalized(snapshot: ObservedSchemaSnapshot): ObservedSchemaSnapshot {
|
||||
const tables = new Map<string, ObservedCatalogTable>();
|
||||
for (const table of snapshot.tables) {
|
||||
if (tables.has(table.name)) throw new CatalogConnectorError("Schema introspection returned duplicate tables");
|
||||
tables.set(table.name, table);
|
||||
}
|
||||
const columns = new Map<string, ObservedCatalogColumn>();
|
||||
for (const column of snapshot.columns) {
|
||||
const key = `${column.tableName}\u0000${column.name}`;
|
||||
if (columns.has(key)) throw new CatalogConnectorError("Schema introspection returned duplicate columns");
|
||||
columns.set(key, column);
|
||||
}
|
||||
const relationships = new Map<string, ObservedCatalogRelationship>();
|
||||
for (const relationship of snapshot.relationships) {
|
||||
const key = `${relationship.sourceTableName}\u0000${relationship.constraintName}`;
|
||||
if (relationships.has(key)) throw new CatalogConnectorError("Schema introspection returned duplicate relationships");
|
||||
relationships.set(key, {
|
||||
...relationship,
|
||||
columns: [...relationship.columns].sort((a, b) => a.position - b.position),
|
||||
});
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
capabilities: snapshot.capabilities,
|
||||
tables: [...tables.values()].sort((a, b) => a.name.localeCompare(b.name)),
|
||||
columns: [...columns.values()].sort((a, b) => (
|
||||
a.tableName.localeCompare(b.tableName) || a.ordinalPosition - b.ordinalPosition
|
||||
)),
|
||||
relationships: [...relationships.values()].sort((a, b) => (
|
||||
a.sourceTableName.localeCompare(b.sourceTableName) || a.constraintName.localeCompare(b.constraintName)
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
export class ConcreteCatalogSchemaIntrospector implements CatalogSchemaIntrospector {
|
||||
constructor(
|
||||
private readonly postgres: CatalogPostgresAccess,
|
||||
private readonly secretStore: WorkspaceSecretStore,
|
||||
) {}
|
||||
|
||||
async scan(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot> {
|
||||
return database.binding.transport === "rest_api"
|
||||
? await this.scanRest(database, signal, progress)
|
||||
: await this.scanPostgres(database, signal, progress);
|
||||
}
|
||||
|
||||
private async scanPostgres(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot> {
|
||||
await progress?.("connecting");
|
||||
const client = await this.postgres.connect(database, signal);
|
||||
try {
|
||||
const schema = await client.query(
|
||||
"SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = $1) AS present",
|
||||
[database.schema],
|
||||
);
|
||||
if (schema.rows[0]?.present !== true) throw new CatalogConnectorError("Database schema is unavailable");
|
||||
|
||||
await progress?.("scanning_tables");
|
||||
const tableResult = await client.query(
|
||||
`SELECT c.relname AS name, d.description AS source_comment
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0
|
||||
WHERE c.relkind IN ('r', 'p') AND n.nspname = $1
|
||||
ORDER BY c.relname`,
|
||||
[database.schema],
|
||||
);
|
||||
const tables: ObservedCatalogTable[] = tableResult.rows.map((row) => ({
|
||||
name: String(row.name),
|
||||
sourceComment: textOrNull(row.source_comment),
|
||||
}));
|
||||
await progress?.("scanning_tables", { tables: tables.length });
|
||||
|
||||
await progress?.("scanning_columns", { tables: tables.length });
|
||||
const columnResult = await client.query(
|
||||
`SELECT c.relname AS table_name,
|
||||
a.attname AS name,
|
||||
a.attnum::integer AS ordinal_position,
|
||||
pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||
NOT a.attnotnull AS is_nullable,
|
||||
pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
|
||||
pk.position AS primary_key_position,
|
||||
d.description AS source_comment
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid
|
||||
LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid = c.oid AND ad.adnum = a.attnum
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT key.ordinality::integer AS position
|
||||
FROM pg_catalog.pg_index i
|
||||
CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS key(attnum, ordinality)
|
||||
WHERE i.indrelid = c.oid AND i.indisprimary AND key.attnum = a.attnum
|
||||
LIMIT 1
|
||||
) pk ON true
|
||||
WHERE c.relkind IN ('r', 'p')
|
||||
AND n.nspname = $1
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
ORDER BY c.relname, a.attnum`,
|
||||
[database.schema],
|
||||
);
|
||||
const columns: ObservedCatalogColumn[] = columnResult.rows.map((row) => ({
|
||||
tableName: String(row.table_name),
|
||||
name: String(row.name),
|
||||
ordinalPosition: Number(row.ordinal_position),
|
||||
dataType: String(row.data_type),
|
||||
isNullable: row.is_nullable === true,
|
||||
defaultExpression: textOrNull(row.default_expression),
|
||||
primaryKeyPosition: row.primary_key_position === null || row.primary_key_position === undefined
|
||||
? null
|
||||
: Number(row.primary_key_position),
|
||||
sourceComment: textOrNull(row.source_comment),
|
||||
}));
|
||||
await progress?.("scanning_columns", { tables: tables.length, columns: columns.length });
|
||||
|
||||
await progress?.("scanning_relationships", { tables: tables.length, columns: columns.length });
|
||||
const relationshipResult = await client.query(
|
||||
`SELECT con.conname AS constraint_name,
|
||||
source_table.relname AS source_table_name,
|
||||
target_table.relname AS target_table_name,
|
||||
con.confupdtype AS update_action,
|
||||
con.confdeltype AS delete_action,
|
||||
con.condeferrable AS deferrable,
|
||||
con.condeferred AS initially_deferred,
|
||||
source_key.ordinality::integer AS position,
|
||||
source_column.attname AS source_column_name,
|
||||
target_column.attname AS target_column_name
|
||||
FROM pg_catalog.pg_constraint con
|
||||
JOIN pg_catalog.pg_class source_table ON source_table.oid = con.conrelid
|
||||
JOIN pg_catalog.pg_namespace source_namespace ON source_namespace.oid = source_table.relnamespace
|
||||
JOIN pg_catalog.pg_class target_table ON target_table.oid = con.confrelid
|
||||
JOIN pg_catalog.pg_namespace target_namespace ON target_namespace.oid = target_table.relnamespace
|
||||
JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) ON true
|
||||
JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS target_key(attnum, ordinality)
|
||||
ON target_key.ordinality = source_key.ordinality
|
||||
JOIN pg_catalog.pg_attribute source_column
|
||||
ON source_column.attrelid = source_table.oid AND source_column.attnum = source_key.attnum
|
||||
JOIN pg_catalog.pg_attribute target_column
|
||||
ON target_column.attrelid = target_table.oid AND target_column.attnum = target_key.attnum
|
||||
WHERE con.contype = 'f'
|
||||
AND source_namespace.nspname = $1
|
||||
AND target_namespace.nspname = $1
|
||||
ORDER BY source_table.relname, con.conname, source_key.ordinality`,
|
||||
[database.schema],
|
||||
);
|
||||
const relationshipMap = new Map<string, ObservedCatalogRelationship>();
|
||||
for (const row of relationshipResult.rows) {
|
||||
const sourceTableName = String(row.source_table_name);
|
||||
const constraintName = String(row.constraint_name);
|
||||
const key = `${sourceTableName}\u0000${constraintName}`;
|
||||
const current = relationshipMap.get(key) ?? {
|
||||
constraintName,
|
||||
sourceTableName,
|
||||
targetTableName: String(row.target_table_name),
|
||||
updateRule: actionRule(row.update_action),
|
||||
deleteRule: actionRule(row.delete_action),
|
||||
deferrable: row.deferrable === true,
|
||||
initiallyDeferred: row.initially_deferred === true,
|
||||
columns: [],
|
||||
};
|
||||
current.columns.push({
|
||||
position: Number(row.position),
|
||||
sourceColumnName: String(row.source_column_name),
|
||||
targetColumnName: String(row.target_column_name),
|
||||
});
|
||||
relationshipMap.set(key, current);
|
||||
}
|
||||
const relationships = [...relationshipMap.values()];
|
||||
await progress?.("scanning_relationships", {
|
||||
tables: tables.length,
|
||||
columns: columns.length,
|
||||
relationships: relationships.length,
|
||||
});
|
||||
return normalized({
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables,
|
||||
columns,
|
||||
relationships,
|
||||
});
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
}
|
||||
|
||||
private async scanRest(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot> {
|
||||
await progress?.("connecting");
|
||||
const auth = database.binding.restAuth ?? "bearer";
|
||||
const materialized = this.secretStore.materialize(
|
||||
database.workspaceId,
|
||||
auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
|
||||
);
|
||||
try {
|
||||
const headers: Record<string, string> = { "content-type": "application/json" };
|
||||
if (auth !== "none") {
|
||||
const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
|
||||
if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
|
||||
const credential = (await readFile(credentialFile, "utf8")).trim();
|
||||
if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
|
||||
else headers["x-api-key"] = credential;
|
||||
}
|
||||
const baseUrl = required(database.binding.baseUrl).replace(/\/+$/, "");
|
||||
const response = await fetch(`${baseUrl}/rpc/schema_snapshot`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ schema_name: database.schema }),
|
||||
signal,
|
||||
});
|
||||
let body: unknown;
|
||||
if (response.ok) {
|
||||
body = await response.json();
|
||||
} else if (response.status === 404) {
|
||||
const fallback = await fetch(`${baseUrl}/rpc/run_query`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ query_text: restSnapshotQuery(database.schema) }),
|
||||
signal,
|
||||
});
|
||||
if (!fallback.ok) throw new CatalogSchemaCapabilityUnavailableError("schema_snapshot");
|
||||
const rows: unknown = await fallback.json();
|
||||
if (!Array.isArray(rows) || rows.length !== 1) {
|
||||
throw new CatalogConnectorError("REST schema snapshot fallback is invalid");
|
||||
}
|
||||
body = rows[0];
|
||||
} else {
|
||||
throw new CatalogSchemaCapabilityUnavailableError("schema_snapshot");
|
||||
}
|
||||
const parsed = restSnapshotSchema.safeParse(body);
|
||||
if (!parsed.success) throw new CatalogConnectorError("REST schema snapshot is invalid");
|
||||
const snapshot = normalized(parsed.data);
|
||||
await progress?.("scanning_tables", { tables: snapshot.tables.length });
|
||||
await progress?.("scanning_columns", { tables: snapshot.tables.length, columns: snapshot.columns.length });
|
||||
await progress?.("scanning_relationships", {
|
||||
tables: snapshot.tables.length,
|
||||
columns: snapshot.columns.length,
|
||||
relationships: snapshot.relationships.length,
|
||||
});
|
||||
return snapshot;
|
||||
} catch (error) {
|
||||
if (error instanceof CatalogConnectorError) throw error;
|
||||
throw new CatalogConnectorError("REST schema introspection failed");
|
||||
} finally {
|
||||
materialized.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
export const CATALOG_SECRET_IDS = {
|
||||
password: "catalog.dwh.password",
|
||||
apiKey: "catalog.dwh.api_key",
|
||||
sshPrivateKey: "catalog.dwh.ssh_private_key",
|
||||
sshPrivateKeyPassphrase: "catalog.dwh.ssh_private_key_passphrase",
|
||||
sshKnownHosts: "catalog.dwh.ssh_known_hosts",
|
||||
tlsCa: "catalog.dwh.tls_ca",
|
||||
} as const;
|
||||
|
||||
export type CatalogSecretName = keyof typeof CATALOG_SECRET_IDS;
|
||||
@@ -1,254 +0,0 @@
|
||||
import { z } from "zod";
|
||||
import type { MetadataGenerationModels } from "./metadata-generation-models.js";
|
||||
import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult, ModelCompletionUsage } from "./model-completer.js";
|
||||
import type {
|
||||
CatalogColumn,
|
||||
CatalogRepository,
|
||||
CatalogTable,
|
||||
SensitiveDataSuggestionScope,
|
||||
} from "./types.js";
|
||||
|
||||
export type { SensitiveDataSuggestionScope } from "./types.js";
|
||||
|
||||
// The helper accepts at most 64 KiB per message. Keep the same safety margin used by
|
||||
// Description Generation so UTF-8 structural metadata never reaches that hard limit.
|
||||
const MAX_USER_MESSAGE_BYTES = 60 * 1024;
|
||||
// Preserve ThothAI's proven completion granularity: small batches keep generation time and
|
||||
// structured-output accuracy predictable even when the helper byte limit would allow much more.
|
||||
const MAX_COLUMNS_PER_BATCH = 10;
|
||||
const responseSchema = z.object({
|
||||
suggestions: z.array(z.object({
|
||||
columnId: z.uuid(),
|
||||
sensitive: z.boolean(),
|
||||
}).strict()),
|
||||
}).strict();
|
||||
|
||||
interface StructuralColumn {
|
||||
columnId: string;
|
||||
tableId: string;
|
||||
table: string;
|
||||
column: string;
|
||||
dataType: string;
|
||||
nullable: boolean;
|
||||
primaryKey: boolean;
|
||||
foreignKey: boolean;
|
||||
version: number;
|
||||
currentSensitive: boolean;
|
||||
}
|
||||
|
||||
export interface SensitiveDataSuggestion {
|
||||
columnId: string;
|
||||
tableId: string;
|
||||
tableName: string;
|
||||
columnName: string;
|
||||
version: number;
|
||||
currentSensitive: boolean;
|
||||
sensitive: boolean;
|
||||
}
|
||||
|
||||
export class SensitiveDataSuggestionTargetNotFoundError extends Error {
|
||||
constructor(readonly target: "database" | "table" | "column") {
|
||||
super(`${target} not found`);
|
||||
this.name = "SensitiveDataSuggestionTargetNotFoundError";
|
||||
}
|
||||
}
|
||||
|
||||
export class SensitiveDataSuggestionDuplicateTargetIdsError extends Error {
|
||||
constructor() {
|
||||
super("sensitive-data suggestion target IDs must be unique");
|
||||
this.name = "SensitiveDataSuggestionDuplicateTargetIdsError";
|
||||
}
|
||||
}
|
||||
|
||||
export class SensitiveDataSuggestionNoEligibleColumnsError extends Error {
|
||||
constructor(readonly scope: SensitiveDataSuggestionScope) {
|
||||
super("selected scope has no catalog columns");
|
||||
this.name = "SensitiveDataSuggestionNoEligibleColumnsError";
|
||||
}
|
||||
}
|
||||
|
||||
export class SensitiveDataSuggestionPayloadTooLargeError extends Error {
|
||||
constructor() {
|
||||
super("sensitive-data suggestion structural metadata is too large");
|
||||
this.name = "SensitiveDataSuggestionPayloadTooLargeError";
|
||||
}
|
||||
}
|
||||
|
||||
export class SensitiveDataSuggestionInvalidResponseError extends Error {
|
||||
constructor() {
|
||||
super("sensitive-data suggestion response is invalid");
|
||||
this.name = "SensitiveDataSuggestionInvalidResponseError";
|
||||
}
|
||||
}
|
||||
|
||||
function userContent(
|
||||
database: { databaseName: string; schema: string },
|
||||
columns: readonly StructuralColumn[],
|
||||
): string {
|
||||
return JSON.stringify({
|
||||
database: database.databaseName,
|
||||
schema: database.schema,
|
||||
columns: columns.map((column) => ({
|
||||
columnId: column.columnId,
|
||||
table: column.table,
|
||||
column: column.column,
|
||||
dataType: column.dataType,
|
||||
nullable: column.nullable,
|
||||
primaryKey: column.primaryKey,
|
||||
foreignKey: column.foreignKey,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
function batchesFor(
|
||||
database: { databaseName: string; schema: string },
|
||||
columns: readonly StructuralColumn[],
|
||||
): StructuralColumn[][] {
|
||||
const batches: StructuralColumn[][] = [];
|
||||
let current: StructuralColumn[] = [];
|
||||
for (const column of columns) {
|
||||
if (current.length === MAX_COLUMNS_PER_BATCH) {
|
||||
batches.push(current);
|
||||
current = [];
|
||||
}
|
||||
const candidate = [...current, column];
|
||||
if (Buffer.byteLength(userContent(database, candidate), "utf8") <= MAX_USER_MESSAGE_BYTES) {
|
||||
current = candidate;
|
||||
continue;
|
||||
}
|
||||
if (current.length === 0) throw new SensitiveDataSuggestionPayloadTooLargeError();
|
||||
batches.push(current);
|
||||
current = [column];
|
||||
if (Buffer.byteLength(userContent(database, current), "utf8") > MAX_USER_MESSAGE_BYTES) {
|
||||
throw new SensitiveDataSuggestionPayloadTooLargeError();
|
||||
}
|
||||
}
|
||||
if (current.length > 0) batches.push(current);
|
||||
return batches;
|
||||
}
|
||||
|
||||
function structuralColumn(table: CatalogTable, column: CatalogColumn): StructuralColumn {
|
||||
return {
|
||||
columnId: column.id,
|
||||
tableId: table.id,
|
||||
table: table.name,
|
||||
column: column.name,
|
||||
dataType: column.dataType,
|
||||
nullable: column.isNullable,
|
||||
primaryKey: column.isPrimaryKey,
|
||||
foreignKey: column.isForeignKey,
|
||||
version: column.version,
|
||||
currentSensitive: column.sensitive,
|
||||
};
|
||||
}
|
||||
|
||||
const systemMessage: ModelCompletionMessage = {
|
||||
role: "system",
|
||||
content: [
|
||||
"Classify whether each database column is likely to contain sensitive source values.",
|
||||
"Use only the supplied structural metadata. Return strict JSON with this exact shape:",
|
||||
'{"suggestions":[{"columnId":"uuid","sensitive":true}]}',
|
||||
"Return every supplied column exactly once. Do not add explanations or markdown.",
|
||||
].join("\n"),
|
||||
};
|
||||
|
||||
export class SensitiveDataSuggester {
|
||||
constructor(
|
||||
private readonly repository: CatalogRepository,
|
||||
private readonly models: MetadataGenerationModels,
|
||||
private readonly completer: ModelCompleter,
|
||||
) {}
|
||||
|
||||
private async selectColumns(
|
||||
databaseId: string,
|
||||
scope: SensitiveDataSuggestionScope,
|
||||
targetIds: readonly string[],
|
||||
): Promise<StructuralColumn[]> {
|
||||
if (new Set(targetIds).size !== targetIds.length) {
|
||||
throw new SensitiveDataSuggestionDuplicateTargetIdsError();
|
||||
}
|
||||
const tables = await this.repository.listTables(databaseId);
|
||||
const tableIds = new Set(targetIds);
|
||||
const selectedTables = scope === "selected_tables"
|
||||
? tables.filter((table) => tableIds.has(table.id))
|
||||
: tables;
|
||||
if (scope === "selected_tables" && selectedTables.length !== targetIds.length) {
|
||||
throw new SensitiveDataSuggestionTargetNotFoundError("table");
|
||||
}
|
||||
|
||||
const columns = (await Promise.all(selectedTables.map(async (table) => (
|
||||
(await this.repository.listColumns(databaseId, table.id)).map((column) => (
|
||||
structuralColumn(table, column)
|
||||
))
|
||||
)))).flat();
|
||||
const columnIds = new Set(targetIds);
|
||||
const selectedColumns = scope === "selected_columns"
|
||||
? columns.filter((column) => columnIds.has(column.columnId))
|
||||
: columns;
|
||||
if (scope === "selected_columns" && selectedColumns.length !== targetIds.length) {
|
||||
throw new SensitiveDataSuggestionTargetNotFoundError("column");
|
||||
}
|
||||
if (selectedColumns.length === 0) {
|
||||
throw new SensitiveDataSuggestionNoEligibleColumnsError(scope);
|
||||
}
|
||||
return selectedColumns;
|
||||
}
|
||||
|
||||
async suggest(
|
||||
databaseId: string,
|
||||
modelId: string,
|
||||
scope: SensitiveDataSuggestionScope,
|
||||
targetIds: readonly string[],
|
||||
signal: AbortSignal,
|
||||
onPrepared?: (total: number) => void | Promise<void>,
|
||||
onProgress?: (processed: number, suggestions: readonly SensitiveDataSuggestion[]) => void | Promise<void>,
|
||||
onUsage?: (usage: ModelCompletionUsage) => void | Promise<void>,
|
||||
): Promise<readonly SensitiveDataSuggestion[]> {
|
||||
const database = await this.repository.get(databaseId);
|
||||
if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
|
||||
const columns = await this.selectColumns(databaseId, scope, targetIds);
|
||||
await onPrepared?.(columns.length);
|
||||
const model = this.models.resolve(modelId);
|
||||
const suggestions: SensitiveDataSuggestion[] = [];
|
||||
|
||||
for (const batch of batchesFor(database, columns)) {
|
||||
let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
|
||||
for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
|
||||
const completion = await this.completer.complete({
|
||||
model,
|
||||
signal,
|
||||
messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
|
||||
});
|
||||
const result: ModelCompletionResult = typeof completion === "string"
|
||||
? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
|
||||
: completion;
|
||||
await onUsage?.(result.usage);
|
||||
const content = result.content;
|
||||
try {
|
||||
const parsed = responseSchema.parse(JSON.parse(content));
|
||||
const expected = new Set(batch.map((column) => column.columnId));
|
||||
const candidate = new Map(parsed.suggestions.map((suggestion) => [suggestion.columnId, suggestion]));
|
||||
if (candidate.size !== parsed.suggestions.length
|
||||
|| candidate.size !== expected.size
|
||||
|| [...candidate.keys()].some((columnId) => !expected.has(columnId))) {
|
||||
throw new SensitiveDataSuggestionInvalidResponseError();
|
||||
}
|
||||
received = candidate;
|
||||
} catch {
|
||||
if (attempt === 1) throw new SensitiveDataSuggestionInvalidResponseError();
|
||||
}
|
||||
}
|
||||
suggestions.push(...batch.map((column) => ({
|
||||
columnId: column.columnId,
|
||||
tableId: column.tableId,
|
||||
tableName: column.table,
|
||||
columnName: column.column,
|
||||
version: column.version,
|
||||
currentSensitive: column.currentSensitive,
|
||||
sensitive: received!.get(column.columnId)!.sensitive,
|
||||
})));
|
||||
await onProgress?.(suggestions.length, suggestions.slice(-batch.length));
|
||||
}
|
||||
return suggestions;
|
||||
}
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
import type {
|
||||
SensitiveDataSuggestion,
|
||||
} from "./sensitive-data-suggester.js";
|
||||
import {
|
||||
SensitiveDataSuggester,
|
||||
SensitiveDataSuggestionTargetNotFoundError,
|
||||
} from "./sensitive-data-suggester.js";
|
||||
import type {
|
||||
CatalogRepository,
|
||||
SensitiveDataSuggestionRun,
|
||||
SensitiveDataSuggestionScope,
|
||||
} from "./types.js";
|
||||
import type { ModelCompletionUsage } from "./model-completer.js";
|
||||
|
||||
const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
|
||||
const failedMessage = "Sensitive-field suggestion generation failed.";
|
||||
|
||||
export interface SensitiveDataSuggestionRunResult {
|
||||
suggestions: readonly SensitiveDataSuggestion[];
|
||||
run: SensitiveDataSuggestionRun;
|
||||
}
|
||||
|
||||
export class SensitiveDataSuggestionRunner {
|
||||
constructor(
|
||||
private readonly repository: CatalogRepository,
|
||||
private readonly suggester: SensitiveDataSuggester,
|
||||
) {}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!(await this.repository.available())) return;
|
||||
const interrupted = await this.repository.interruptActiveSensitiveDataSuggestionRuns(
|
||||
interruptedMessage,
|
||||
);
|
||||
for (const run of interrupted) {
|
||||
await this.repository.appendSensitiveDataSuggestionEvent(
|
||||
run.id,
|
||||
"warning",
|
||||
interruptedMessage,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async run(
|
||||
databaseId: string,
|
||||
modelId: string,
|
||||
scope: SensitiveDataSuggestionScope,
|
||||
targetIds: readonly string[],
|
||||
signal: AbortSignal,
|
||||
): Promise<SensitiveDataSuggestionRunResult> {
|
||||
if (!(await this.repository.get(databaseId))) {
|
||||
throw new SensitiveDataSuggestionTargetNotFoundError("database");
|
||||
}
|
||||
const started = await this.repository.createSensitiveDataSuggestionRun(
|
||||
databaseId,
|
||||
scope,
|
||||
modelId,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.repository.appendSensitiveDataSuggestionEvent(
|
||||
started.id,
|
||||
"info",
|
||||
"Sensitive-field suggestion generation started.",
|
||||
);
|
||||
const suggestions = await this.suggester.suggest(
|
||||
databaseId,
|
||||
modelId,
|
||||
scope,
|
||||
targetIds,
|
||||
signal,
|
||||
async (total) => {
|
||||
const prepared = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
|
||||
total,
|
||||
});
|
||||
if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||
},
|
||||
async (processed, batch) => {
|
||||
const suggestedSensitive = batch.filter((suggestion) => suggestion.sensitive).length;
|
||||
const suggestedNonSensitive = batch.length - suggestedSensitive;
|
||||
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
|
||||
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||
const progress = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
|
||||
suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
|
||||
suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
|
||||
});
|
||||
if (!progress) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||
await this.repository.appendSensitiveDataSuggestionEvent(
|
||||
started.id,
|
||||
"info",
|
||||
`Classified ${processed} of ${progress.total} columns.`,
|
||||
);
|
||||
},
|
||||
async (usage: ModelCompletionUsage) => {
|
||||
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
|
||||
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||
await this.repository.updateSensitiveDataSuggestionRun(started.id, {
|
||||
inputTokens: current.inputTokens + usage.input,
|
||||
cacheReadTokens: current.cacheReadTokens + usage.cacheRead,
|
||||
outputTokens: current.outputTokens + usage.output,
|
||||
});
|
||||
},
|
||||
);
|
||||
const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
|
||||
const suggestedNonSensitive = suggestions.length - suggestedSensitive;
|
||||
await this.repository.appendSensitiveDataSuggestionEvent(
|
||||
started.id,
|
||||
"info",
|
||||
`Sensitive-field suggestion generation completed for ${suggestions.length} column${
|
||||
suggestions.length === 1 ? "" : "s"
|
||||
}.`,
|
||||
);
|
||||
const completed = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
|
||||
status: "completed",
|
||||
total: suggestions.length,
|
||||
suggestedSensitive,
|
||||
suggestedNonSensitive,
|
||||
finishedAt: new Date().toISOString(),
|
||||
errorSummary: null,
|
||||
});
|
||||
if (!completed) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||
return { suggestions, run: completed };
|
||||
} catch (error) {
|
||||
await this.repository.updateSensitiveDataSuggestionRun(started.id, {
|
||||
status: "failed",
|
||||
finishedAt: new Date().toISOString(),
|
||||
errorSummary: failedMessage,
|
||||
}).catch(() => undefined);
|
||||
await this.repository.appendSensitiveDataSuggestionEvent(
|
||||
started.id,
|
||||
"error",
|
||||
failedMessage,
|
||||
).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user