diff --git a/README.md b/README.md new file mode 100644 index 00000000..963169a6 --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# ThothII + +ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fastify/Pi/`tht` +core. The portable deployment runs exactly two application services; data services remain +external in this profile. + +## Docker Compose: external services + +Requirements: Docker Engine with Compose v2 and reachable DWH, vector, and embeddings +services. + +1. Copy `deploy/env.example` to `deploy/.env` and fill in runtime credentials. The `.env` + file is gitignored and is read only when the container starts; secrets are never copied + into either image. +2. Add or edit YAML workspace descriptors under `deploy/workspaces/`. These files are mounted + read-only. Use relative `roots`; they resolve beneath `/data/workspaces/`. +3. Start the external-service profile: + + ```sh + docker compose --profile external up --build --wait + ``` + +4. Open . Set `THOTH_HTTP_PORT` before starting to use another host + port. + +Application state, including settings, sessions, artifacts, and indexes, lives in the named +`thoth_data` volume mounted at `/data`. `docker compose down` keeps that volume. Only an +explicit destructive command such as `docker compose down --volumes` removes it. + +The frontend depends on the core health check and proxies `/health` and `/api/*` to it. The +application health endpoint intentionally checks process readiness only; external dependency +diagnostics are exposed by `tht doctor` and do not prevent the UI from starting. + +Run the end-to-end packaging check with: + +```sh +./scripts/docker-smoke.sh +``` + +The smoke script validates Compose, builds and waits for both services, checks health through +the frontend, verifies SSE response headers, restarts the core, and confirms `/data` survives. +Its cleanup preserves the named volume. diff --git a/backend/src/routes/sessions.ts b/backend/src/routes/sessions.ts index cfc29212..479c0bbc 100644 --- a/backend/src/routes/sessions.ts +++ b/backend/src/routes/sessions.ts @@ -77,6 +77,9 @@ export function sessionRoutes( "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Credentials": "true", }); + // Send the handshake immediately. Without this, Node waits for the first event body and + // proxies/clients cannot establish an idle SSE subscription or inspect its headers. + reply.raw.flushHeaders(); const send = (event: string, data: object) => reply.raw.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); const off = d.hub.subscribe(id, send, rt?.bridge.pendingWidget() ?? null); req.raw.on("close", off); diff --git a/backend/test/health.test.ts b/backend/test/health.test.ts index d94c24c4..628445b3 100644 --- a/backend/test/health.test.ts +++ b/backend/test/health.test.ts @@ -2,9 +2,33 @@ import { test, expect } from "vitest"; import { buildApp } from "../src/app.js"; import { loadConfig } from "../src/config.js"; -test("GET /health ritorna ok", async () => { +test("GET /health reports process readiness without external services", async () => { const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/tmp/h" })); const res = await app.inject({ method: "GET", url: "/health" }); expect(res.statusCode).toBe(200); + expect(res.headers["content-type"]).toContain("application/json"); expect(res.json()).toEqual({ status: "ok" }); }); + +test("SSE response headers are flushed before the first event", async () => { + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/tmp/h" })); + await app.listen({ port: 0, host: "127.0.0.1" }); + const port = (app.server.address() as { port: number }).port; + const controller = new AbortController(); + + try { + const response = await Promise.race([ + fetch(`http://127.0.0.1:${port}/sessions/header-probe/events`, { + signal: controller.signal, + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error("SSE headers were not flushed")), 250), + ), + ]); + expect(response.headers.get("content-type")).toContain("text/event-stream"); + expect(response.headers.get("cache-control")).toBe("no-cache"); + } finally { + controller.abort(); + await app.close(); + } +}); diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..7b0cc806 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,47 @@ +name: thothii + +services: + core: + profiles: [external] + build: + context: . + dockerfile: docker/core.Dockerfile + env_file: + - path: deploy/.env + required: false + environment: + THT_DATA_ROOT: /data + SETTINGS_FILE: /data/settings/settings.json + volumes: + - thoth_data:/data + - ./deploy/workspaces:/app/harness/workspaces:ro + healthcheck: + test: [CMD, curl, --fail, --silent, http://127.0.0.1:8787/health] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + + frontend: + profiles: [external] + build: + context: . + dockerfile: docker/frontend.Dockerfile + environment: + BACKEND_BASE_URL: /api + ports: + - "${THOTH_HTTP_PORT:-8080}:8080" + depends_on: + core: + condition: service_healthy + healthcheck: + test: [CMD, wget, --quiet, --spider, http://127.0.0.1:8080/health] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + restart: unless-stopped + +volumes: + thoth_data: diff --git a/deploy/env.example b/deploy/env.example new file mode 100644 index 00000000..f51bb870 --- /dev/null +++ b/deploy/env.example @@ -0,0 +1,28 @@ +# Copy this file to deploy/.env. Never commit deploy/.env or real credentials. +# Compose passes these values to the core container at runtime; images contain no secrets. + +# Optional application defaults +PI_PROVIDER= +PI_MODEL= +PI_THINKING= +MAX_PI_PROCESSES=4 +AUTH_MODE=none + +# External DWH (example workspace uses the HTTP adapters) +THT_DB_NAME= +THT_DWH_REST_URL= +THT_DWH_API_KEY= + +# External vector service. Use a distinct write key where the service supports one. +THT_VEC_REST_URL= +THT_VEC_API_KEY= +THT_VEC_WRITE_API_KEY= + +# External embeddings service +THT_OLLAMA_URL= + +# Evidence source visible inside the persistent data volume +THT_DOCS_ROOT=/data/workspaces/example/evidence-source + +# Optional CA file mounted separately by an operator, for example via a Compose override. +THT_SSL_CA= diff --git a/deploy/workspaces/example.yaml b/deploy/workspaces/example.yaml new file mode 100644 index 00000000..10c1a186 --- /dev/null +++ b/deploy/workspaces/example.yaml @@ -0,0 +1,69 @@ +language: en + +dwh: + type: thoth_rest + database: + database: ${THT_DB_NAME} + schema: datawarehouse + endpoint: + base_url: ${THT_DWH_REST_URL} + api_key: ${THT_DWH_API_KEY} + ssl_ca: ${THT_SSL_CA} + +# Relative logical roots are resolved beneath /data/workspaces/example. +roots: + artifacts: artifacts + indexes: indexes + sessions: sessions + +examples: + max_per_column: 10 + +lsh: + signature_size: 64 + n_gram: 3 + threshold: 0.5 + max_values_per_column: 1000 + +eligibility: + max_declared_len: 128 + max_avg_length: 40 + max_sampled_len: 200 + ignore_columns: [etl_last_update] + +evidence: + source_root: ${THT_DOCS_ROOT} + evidence_dir: evidence + +embeddings: + base_url: ${THT_OLLAMA_URL} + model: nomic-embed-text-v2-moe + dim: 768 + batch_size: 32 + +vectors: + type: thoth_vector_http + reader: + base_url: ${THT_VEC_REST_URL} + api_key: ${THT_VEC_API_KEY} + ssl_ca: ${THT_SSL_CA} + writer: + base_url: ${THT_VEC_REST_URL} + api_key: ${THT_VEC_WRITE_API_KEY} + ssl_ca: ${THT_SSL_CA} + +vector: + max_chunk_chars: 4000 + +search: + rrf_k: 60 + top_schema_tables: 12 + schema_chunk_pool: 150 + +execution: + allow: [cte_test, explain, preview, aggregate, export] + max_preview_rows: 10 + max_export_rows: 100000 + statement_timeout_ms: 30000 + warn_execution_ms: 5000 + max_aggregate_cells: 20 diff --git a/scripts/docker-smoke.sh b/scripts/docker-smoke.sh new file mode 100755 index 00000000..42081604 --- /dev/null +++ b/scripts/docker-smoke.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +compose="docker compose --profile external" +marker="smoke-$(date +%s)-$$" +headers="" +# Avoid colliding with a developer's existing service. Production/developer Compose still +# defaults to 8080; a published port of 0 asks Docker for a free ephemeral smoke port. +export THOTH_HTTP_PORT=${THOTH_HTTP_PORT:-0} + +cleanup() { + if [ -n "$headers" ]; then rm -f "$headers"; fi + # Deliberately omit --volumes: an ordinary smoke run must preserve user data. + $compose down --remove-orphans >/dev/null 2>&1 || true +} +trap cleanup EXIT HUP INT TERM + +$compose config --quiet +$compose up --build --wait core frontend +published=$($compose port frontend 8080) +http_port=${published##*:} + +curl --fail --silent --show-error "http://127.0.0.1:$http_port/health" >/dev/null + +# The nginx proxy must preserve streaming semantics for the backend SSE endpoint. +headers=$(mktemp) +curl --fail --silent --show-error --max-time 2 --dump-header "$headers" \ + "http://127.0.0.1:$http_port/api/sessions/docker-smoke/events" \ + >/dev/null 2>&1 || status=$? +status=${status:-0} +if [ "$status" -ne 0 ] && [ "$status" -ne 28 ]; then + echo "SSE probe failed with curl status $status" >&2 + exit "$status" +fi +grep -qi '^content-type: text/event-stream' "$headers" +grep -qi '^cache-control: no-cache' "$headers" +rm -f "$headers" +headers="" + +# Write through the named volume, restart the application, and prove persistence. +$compose exec -T core sh -c 'printf "%s\n" "$1" > /data/.compose-smoke-marker' sh "$marker" +$compose restart core +$compose up --wait core frontend +persisted=$($compose exec -T core sh -c 'cat /data/.compose-smoke-marker') +[ "$persisted" = "$marker" ] +$compose exec -T core rm -f /data/.compose-smoke-marker + +curl --fail --silent --show-error "http://127.0.0.1:$http_port/health" >/dev/null +echo "Compose health, SSE proxy, and restart persistence checks passed."