From 7628eaa579409f14aa40660a2b5784032eec6d16 Mon Sep 17 00:00:00 2001 From: mptyl Date: Sun, 12 Jul 2026 19:20:10 +0200 Subject: [PATCH] fix(docker): run real questions through trusted Pi gate --- .env.example | 2 + .gitignore | 2 + backend/src/config/secret-bundle.ts | 5 +- backend/src/pi/pi-process-manager.ts | 12 +++- backend/src/routes/sessions.ts | 9 +-- backend/src/tht/tht-runner.ts | 13 ++-- backend/test/pi-process-manager.test.ts | 11 +++ compose.yaml | 4 ++ deploy/compose.psd-local.yaml.example | 11 +++ deploy/pi/models.json | 18 +++++ deploy/pi/settings.json | 3 + deploy/workspaces/psd.yaml.example | 43 ++++++++++++ docker/core.Dockerfile | 6 +- ...7-12-local-and-server-docker-deployment.md | 70 +++++++++++++++++++ scripts/bootstrap-local-psd-docker-config.sh | 68 ++++++++++++++++++ scripts/test-container-deployment.sh | 3 + 16 files changed, 266 insertions(+), 14 deletions(-) create mode 100644 deploy/compose.psd-local.yaml.example create mode 100644 deploy/pi/models.json create mode 100644 deploy/pi/settings.json create mode 100644 deploy/workspaces/psd.yaml.example create mode 100644 docs/superpowers/plans/2026-07-12-local-and-server-docker-deployment.md create mode 100644 scripts/bootstrap-local-psd-docker-config.sh diff --git a/.env.example b/.env.example index bac64f66..444490bb 100644 --- a/.env.example +++ b/.env.example @@ -17,8 +17,10 @@ PI_THINKING= THT_DB_NAME= THT_DWH_REST_URL= THT_VEC_REST_URL= +THT_VEC_WRITE_REST_URL= THT_OLLAMA_URL= THT_DOCS_ROOT=/data/workspaces/example/evidence-source +THT_PROFILE=server # Local-vector defaults (used by the optional local-vector overlay). THT_VECTOR_DATABASE=thoth diff --git a/.gitignore b/.gitignore index 7b0178fc..85fe9517 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,8 @@ config/ca-chain.pem # ThothII deployment configuration and secret values (keep only the README tracked) deploy/.env +deploy/compose.psd-local.yaml +deploy/workspaces/psd.yaml deploy/secrets/* !deploy/secrets/README.md !deploy/secrets/*.example diff --git a/backend/src/config/secret-bundle.ts b/backend/src/config/secret-bundle.ts index db2ace5b..b46746a3 100644 --- a/backend/src/config/secret-bundle.ts +++ b/backend/src/config/secret-bundle.ts @@ -56,7 +56,10 @@ function unavailable(): Error { return new Error("secret bundle is unavailable") function secureStat(info: Stats, docker: boolean): boolean { const mode = info.mode & 0o777; if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.size > MAX_BUNDLE_BYTES) return false; - if (docker) return info.uid === 0 && mode === 0o444; + if (docker) { + return (info.uid === 0 && mode === 0o444) + || (info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600)); + } return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600); } diff --git a/backend/src/pi/pi-process-manager.ts b/backend/src/pi/pi-process-manager.ts index 8bdec490..70e2d6f3 100644 --- a/backend/src/pi/pi-process-manager.ts +++ b/backend/src/pi/pi-process-manager.ts @@ -44,6 +44,14 @@ export class PiProcessManager { credentialFile: this.cfg.modelApiKeyFile, additions: { THT_SESSION: sessionId, THT_AUTHOR: author }, }); + // The Thoth gate executes the deterministic `tht` CLI as a Pi tool. Give only + // this managed session process the adapter values already loaded by the core + // entrypoint; the generic provider helper continues to scrub them by default. + for (const name of [ + "THT_DWH_API_KEY", "THT_VEC_API_KEY", "THT_VEC_WRITE_API_KEY", "THT_SSL_CA", + ] as const) { + if (process.env[name] !== undefined) env[name] = process.env[name]; + } delete env.THT_DATA_ROOT; if (this.cfg.dataRoot !== undefined) env.THT_DATA_ROOT = this.cfg.dataRoot; // pi 0.73 removed `--approve`: rpc mode is headless and its argv is intentionally minimal. @@ -62,7 +70,7 @@ export class PiProcessManager { async spawnFor( sessionId: string, - o: { provider?: string; model?: string; thinking?: string; author?: string; mode?: "new" | "resume" }, + o: { provider?: string; model?: string; thinking?: string; author?: string; question?: string; mode?: "new" | "resume" }, ): Promise { // Idempotent per session id: tear down any existing runtime for this id // first (before the cap check) so a resume/respawn neither leaks the old @@ -110,7 +118,7 @@ export class PiProcessManager { const message = o.mode === "resume" ? `/riprendi-sessione ${sessionId}` - : `/nuova-domanda "kickoff"`; + : `/nuova-domanda ${JSON.stringify(o.question ?? "")}`; rpc.send({ type: "prompt", message }); return rt; } diff --git a/backend/src/routes/sessions.ts b/backend/src/routes/sessions.ts index 479c0bbc..ca85acd2 100644 --- a/backend/src/routes/sessions.ts +++ b/backend/src/routes/sessions.ts @@ -29,12 +29,13 @@ export function sessionRoutes( model: s.model, thinking: s.thinking, author: getUser(req).id, + question: b.question, }); rt.bridge.onClientEvent((e) => d.hub.publish(id, e.type, e)); return { id }; }); - app.get("/sessions", async () => d.tht.sessionList()); - app.get("/sessions/:id", async (req) => d.tht.sessionShow((req.params as any).id)); + app.get("/sessions", async () => d.tht.sessionList(d.getSettings().workspace)); + app.get("/sessions/:id", async (req) => d.tht.sessionShow((req.params as any).id, d.getSettings().workspace)); app.post("/sessions/:id/response", async (req, reply) => { const id = (req.params as any).id; const rt = d.mgr.get(id); @@ -50,7 +51,7 @@ export function sessionRoutes( }); app.post("/sessions/:id/resume", async (req, reply) => { const id = (req.params as any).id; - const manifest = (await d.tht.sessionShow(id)) as { status?: string; archived?: boolean } | null; + const manifest = (await d.tht.sessionShow(id, d.getSettings().workspace)) as { status?: string; archived?: boolean } | null; if (manifest?.status === "finalized" || manifest?.archived) { return reply.code(409).send({ error: "sessione in sola lettura (finalizzata o archiviata)" }); } @@ -103,7 +104,7 @@ export function sessionRoutes( app.delete("/sessions/:id", async (req, reply) => { const id = (req.params as any).id; d.mgr.teardown(id); // drop any live runtime before deleting on disk - await d.tht.deleteSession(id); + await d.tht.deleteSession(id, d.getSettings().workspace); return reply.code(204).send(); }); app.get("/sessions/:id/documents", async (req) => d.tht.documents((req.params as any).id)); diff --git a/backend/src/tht/tht-runner.ts b/backend/src/tht/tht-runner.ts index 1406a21a..7bf1dd95 100644 --- a/backend/src/tht/tht-runner.ts +++ b/backend/src/tht/tht-runner.ts @@ -109,12 +109,12 @@ export class ThtRunner { return this.json<{ id: string }>(a, o.workspace); } - sessionList() { - return this.json(["session", "list", "--json"]); + sessionList(workspace?: string) { + return this.json(["session", "list", "--json"], workspace); } - sessionShow(id: string) { - return this.json(["session", "show", id, "--json"]); + sessionShow(id: string, workspace?: string) { + return this.json(["session", "show", id, "--json"], workspace); } sqlPreview(id: string, p: { limit?: number; offset?: number }) { @@ -141,7 +141,10 @@ export class ThtRunner { setGroup(id: string, group: string) { return this.ok(["session", "set-group", id, "--group", group]); } archive(id: string) { return this.ok(["session", "archive", id]); } unarchive(id: string) { return this.ok(["session", "unarchive", id]); } - deleteSession(id: string) { return this.ok(["session", "delete", id]); } + async deleteSession(id: string, workspace?: string) { + const { code, stderr } = await this.run(["session", "delete", id], workspace); + if (code !== 0) throw new Error(`tht session delete exit ${code}: ${stderr.trim()}`); + } documents(id: string) { return this.json(["session", "documents", id, "--json"]); } async ollamaEnsure(workspace: string, timeoutSec: number): Promise { diff --git a/backend/test/pi-process-manager.test.ts b/backend/test/pi-process-manager.test.ts index 4314114d..413e2ee4 100644 --- a/backend/test/pi-process-manager.test.ts +++ b/backend/test/pi-process-manager.test.ts @@ -127,6 +127,17 @@ test("spawnFor default (new) mode sends /nuova-domanda", async () => { mgr.teardown("sid-10"); }); +test("spawnFor new mode forwards the real question instead of kickoff", async () => { + const cfg = loadConfig({}); + const child = recordingChild(); + const mgr = new PiProcessManager(cfg, { spawnFn: () => child as any }); + await mgr.spawnFor("sid-question", { question: "pazienti con cardioversione e ILR" }); + const prompt = JSON.parse(child._writes.at(-1)!); + expect(prompt.message).toBe('/nuova-domanda "pazienti con cardioversione e ILR"'); + expect(prompt.message).not.toContain("kickoff"); + mgr.teardown("sid-question"); +}); + test("production spawn uses explicit Pi path and passes portable data root without rewriting PATH", async () => { vi.stubEnv("PATH", "/usr/local/bin:/usr/bin"); vi.stubEnv("PI_PROVIDER_API_KEY", "provider-secret"); diff --git a/compose.yaml b/compose.yaml index 836179b9..72ee8a7d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,7 +20,9 @@ services: THT_DB_NAME: "${THT_DB_NAME:-}" THT_DWH_REST_URL: "${THT_DWH_REST_URL:-}" THT_VEC_REST_URL: "${THT_VEC_REST_URL:-}" + THT_VEC_WRITE_REST_URL: "${THT_VEC_WRITE_REST_URL:-}" THT_OLLAMA_URL: "${THT_OLLAMA_URL:-}" + THT_PROFILE: "${THT_PROFILE:-server}" THT_DOCS_ROOT: "${THT_DOCS_ROOT:-/data/workspaces/example/evidence-source}" THT_SECRETS_FILE: /run/secrets/thothii.secrets THT_DATA_ROOT: /data @@ -31,6 +33,8 @@ services: volumes: - thoth_data:/data - ./deploy/workspaces:/app/harness/workspaces:ro + - ./deploy/pi/models.json:/home/thoth/.pi/agent/models.json:ro + - ./deploy/pi/settings.json:/home/thoth/.pi/agent/settings.json:ro healthcheck: test: [CMD, curl, --fail, --silent, http://127.0.0.1:8787/health] interval: 5s diff --git a/deploy/compose.psd-local.yaml.example b/deploy/compose.psd-local.yaml.example new file mode 100644 index 00000000..0325db47 --- /dev/null +++ b/deploy/compose.psd-local.yaml.example @@ -0,0 +1,11 @@ +services: + core: + environment: + THT_DOCS_ROOT: /data/workspaces/psd + extra_hosts: + - host.docker.internal:host-gateway + volumes: + - ./deploy/workspaces/psd.yaml:/app/harness/config/tht.yaml:ro + - type: bind + source: ${THT_PSD_WORKSPACE_HOST_PATH:?set THT_PSD_WORKSPACE_HOST_PATH} + target: /data/workspaces/psd diff --git a/deploy/pi/models.json b/deploy/pi/models.json new file mode 100644 index 00000000..1b49dac8 --- /dev/null +++ b/deploy/pi/models.json @@ -0,0 +1,18 @@ +{ + "providers": { + "zai": { + "baseUrl": "https://api.z.ai/api/coding/paas/v4", + "api": "openai-completions", + "apiKey": "$ZAI_API_KEY", + "models": [ + { + "id": "glm-5.2", + "name": "GLM-5.2", + "reasoning": true, + "contextWindow": 200000, + "maxTokens": 131072 + } + ] + } + } +} diff --git a/deploy/pi/settings.json b/deploy/pi/settings.json new file mode 100644 index 00000000..bc2fe370 --- /dev/null +++ b/deploy/pi/settings.json @@ -0,0 +1,3 @@ +{ + "defaultProjectTrust": "always" +} diff --git a/deploy/workspaces/psd.yaml.example b/deploy/workspaces/psd.yaml.example new file mode 100644 index 00000000..4b9e52f3 --- /dev/null +++ b/deploy/workspaces/psd.yaml.example @@ -0,0 +1,43 @@ +language: it + +dwh: + type: thoth_rest + database: + database: ${THT_DB_NAME} + schema: datawarehouse + endpoint: + base_url: ${THT_DWH_REST_URL} + api_key: ${THT_DWH_API_KEY} + ssl_ca: ${THT_SSL_CA} + +vectors: + type: thoth_vector_http + reader: + base_url: ${THT_VEC_REST_URL} + api_key: ${THT_VEC_API_KEY} + ssl_ca: ${THT_SSL_CA} + writer: + base_url: ${THT_VEC_WRITE_REST_URL} + api_key: ${THT_VEC_WRITE_API_KEY} + ssl_ca: ${THT_SSL_CA} + +roots: + artifacts: /data/workspaces/psd/runtime-v2/artifacts + indexes: /data/workspaces/psd/runtime-v2/indexes + sessions: /data/workspaces/psd/sessions + +evidence: + source_root: ${THT_DOCS_ROOT} + evidence_dir: evidence + +embeddings: + base_url: ${THT_OLLAMA_URL} + model: nomic-embed-text-v2-moe + dim: 768 + batch_size: 32 + +execution: + allow: [cte_test, explain, preview, aggregate, export] + max_preview_rows: 10 + max_export_rows: 100000 + statement_timeout_ms: 30000 diff --git a/docker/core.Dockerfile b/docker/core.Dockerfile index a199e489..c66ef573 100644 --- a/docker/core.Dockerfile +++ b/docker/core.Dockerfile @@ -24,8 +24,8 @@ RUN apt-get update \ && apt-get install --yes --no-install-recommends ca-certificates curl \ && rm -rf /var/lib/apt/lists/* \ && useradd --create-home --uid 10001 thoth \ - && mkdir -p /app/backend /app/docker/smoke /data/settings \ - && chown -R thoth:thoth /data + && mkdir -p /app/backend /app/docker/smoke /app/harness/config /data/settings /home/thoth/.pi/agent \ + && chown -R thoth:thoth /app/harness /data /home/thoth/.pi COPY --from=node-runtime /usr/local/bin/node /usr/local/bin/node COPY --from=node-runtime /opt/pi-runtime /opt/pi-runtime @@ -36,11 +36,13 @@ RUN ln -s /opt/pi-runtime/node_modules/.bin/pi /usr/local/bin/pi \ WORKDIR /app COPY harness/ /app/harness/ COPY --from=gate-deps /src/harness/node_modules /app/harness/node_modules +RUN chown -R thoth:thoth /app/harness COPY docker/python-runtime/requirements.lock /app/docker/python-runtime/requirements.lock RUN python -m venv /opt/venv \ && /opt/venv/bin/pip install --no-cache-dir --require-hashes \ --requirement /app/docker/python-runtime/requirements.lock \ && /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation '/app/harness[s3]' \ + && cp /app/harness/workflow.yaml /opt/venv/lib/python3.12/site-packages/workflow.yaml \ && /opt/venv/bin/tht vector migrate --help >/dev/null COPY --from=backend-build /src/backend/dist /app/backend/dist diff --git a/docs/superpowers/plans/2026-07-12-local-and-server-docker-deployment.md b/docs/superpowers/plans/2026-07-12-local-and-server-docker-deployment.md new file mode 100644 index 00000000..f64a5075 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-local-and-server-docker-deployment.md @@ -0,0 +1,70 @@ +# Local and Server Docker Deployment Implementation Plan + +> **For Codex:** execute this plan in the current isolated worktree; keep runtime credentials out of Git. + +**Goal:** Configure and verify a Docker Desktop deployment using GLM 5.2 and the existing PSD workspace, while retaining a portable server deployment contract. + +**Architecture:** The base Compose file builds two applications and consumes only generic environment values and a Docker secret bundle. A tracked GLM Pi registry is mounted read-only in the core container. A Git-ignored local override supplies Mac-specific PSD workspace and CA mounts; server operators supply equivalent server runtime values separately. + +**Tech Stack:** Docker Compose v2, Node 22, Python 3.12, Pi RPC, Fastify, nginx. + +--- + +### Task 1: Add the non-secret GLM Pi registry + +**Files:** +- Create: `deploy/pi/models.json` +- Modify: `docker/core.Dockerfile` +- Modify: `compose.yaml` +- Test: Compose configuration and Pi model discovery + +1. Define the `zai/glm-5.2` OpenAI-compatible model registry without a credential. +2. Create the Pi user configuration directory in the core image and mount the registry read-only. +3. Verify that `get_available_models` returns `zai/glm-5.2` when the bundle supplies the model key. + +### Task 2: Add generic PSD-compatible runtime templates + +**Files:** +- Create: `deploy/workspaces/psd.yaml.example` +- Create: `deploy/compose.psd-local.yaml.example` +- Modify: `deploy/env.example` +- Modify: `README.md` + +1. Define a relative `/data/workspaces/psd` workspace configuration with external REST DWH/vector adapters. +2. Document required non-secret environment values and the local/server boundary. +3. Keep host paths and credential values out of all tracked files. + +### Task 3: Materialize local runtime configuration securely + +**Files (ignored):** +- Create: `.env` +- Create: `deploy/secrets/thothii.secrets` +- Create: `deploy/compose.psd-local.yaml` +- Create: `deploy/workspaces/psd.yaml` + +1. Transfer only required values from the existing local configuration without writing them to logs. +2. Set `PI_PROVIDER=zai`, `PI_MODEL=glm-5.2`, and the Docker Desktop host gateway for Ollama. +3. Bind-mount the PSD workspace and private CA read-only where appropriate; sessions remain writable. +4. Enforce restricted modes on the secret bundle. + +### Task 4: Build and verify the Docker deployment + +**Commands:** +- `docker compose config --quiet` +- `docker compose build` +- `docker compose up -d` +- health/API/model/session smoke checks + +1. Validate rendered Compose configuration without exposing secrets. +2. Build the core and frontend images. +3. Verify secret mount, core and frontend health, and model listing. +4. Start a PSD session using GLM 5.2 and verify Pi emits a workflow event or gate. +5. Capture sanitized diagnostics and stop only disposable test resources; leave the validated local stack running unless it fails. + +### Task 5: Record the deployment result + +**Files:** +- Modify: `README.md` or deployment documentation + +1. Record the exact local startup command and server-equivalent configuration steps. +2. State verified endpoints, model, and session-start result without secret values. diff --git a/scripts/bootstrap-local-psd-docker-config.sh b/scripts/bootstrap-local-psd-docker-config.sh new file mode 100644 index 00000000..1077ccfe --- /dev/null +++ b/scripts/bootstrap-local-psd-docker-config.sh @@ -0,0 +1,68 @@ +#!/bin/sh +set -eu + +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +source_env=${1:-"$root/../../harness/.env"} +workspace=${2:-"$root/../../../tht-workspace-psd"} +auth_file=${3:-"$HOME/.pi/agent/auth.json"} + +value() { + awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); sub(/[[:space:]].*$/, ""); print; exit }' "$source_env" +} +required() { + result=$(value "$1") + [ -n "$result" ] || { echo "missing $1 in local source configuration" >&2; exit 2; } + printf '%s' "$result" +} + +test -f "$source_env" +test -d "$workspace" +test -f "$auth_file" +ca=$(value THT_SSL_CA) +[ -z "$ca" ] || test -f "$ca" +model_key=$(jq -er '.zai.key' "$auth_file") +test -n "$model_key" + +umask 077 +mkdir -p "$root/deploy/secrets" "$root/deploy/workspaces" +cp "$root/deploy/compose.psd-local.yaml.example" "$root/deploy/compose.psd-local.yaml" +cp "$root/deploy/workspaces/psd.yaml.example" "$root/deploy/workspaces/psd.yaml" +cat >"$root/.env" <"$root/deploy/secrets/thothii.secrets" <>"$root/deploy/compose.psd-local.yaml" <>"$root/deploy/secrets/thothii.secrets" +else + printf '%s\n' 'THT_CA=/etc/ssl/certs/ca-certificates.crt' >>"$root/deploy/secrets/thothii.secrets" +fi +chmod 600 "$root/.env" "$root/deploy/secrets/thothii.secrets" +echo "Local PSD Docker configuration materialized without printing secret values." diff --git a/scripts/test-container-deployment.sh b/scripts/test-container-deployment.sh index 9aa907cc..fb0965c1 100755 --- a/scripts/test-container-deployment.sh +++ b/scripts/test-container-deployment.sh @@ -25,6 +25,9 @@ grep -q 'host_ip: 127.0.0.1' "$tmp/base.yaml" grep -q 'AUTH_MODE: none' "$tmp/base.yaml" grep -q 'THOTH_PUBLIC_EXPOSURE: "false"' "$tmp/base.yaml" grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets' "$tmp/base.yaml" +grep -q 'target: /home/thoth/.pi/agent/models.json' "$tmp/base.yaml" +grep -q 'source: .*/deploy/pi/models.json' "$tmp/base.yaml" +grep -q 'target: /home/thoth/.pi/agent/settings.json' "$tmp/base.yaml" if grep -q 'THT_[A-Z0-9_]*_SECRET_FILE:' "$tmp/base.yaml"; then echo "base Compose must not require legacy secret-file variables" >&2 exit 1