From 67d030b24ddb7d4de25eef14687ce11e0e5be255 Mon Sep 17 00:00:00 2001 From: User Date: Sun, 12 Jul 2026 16:49:03 +0200 Subject: [PATCH] feat(deploy): docker images, compose, roles SQL, local workspace - core.Dockerfile: python:3.12-slim + node 22 copied (same bookworm glibc), non-root, tht+pi - frontend.Dockerfile: vite build (env-driven base/assetsDir) + nginx-unprivileged - compose.yaml (embedded, omics_network ext, zero host ports) + docker-compose.dev.yml (standalone) - deploy/sql: thoth_dwh_reader (ro) + thoth_vector_rw (rw) roles - deploy/thothii.env.example + harness/workspaces/local.yaml (direct DWH+vector, 5438) - scripts/docker-smoke.sh; .dockerignore; gitignore deploy secrets - verified: both images build, core health {ok}, config check validates local.yaml --- .dockerignore | 17 ++++++++ .gitignore | 2 + compose.yaml | 48 ++++++++++++++++++++++ deploy/sql/10-dwh-roles.sql | 15 +++++++ deploy/sql/20-vector-roles.sql | 23 +++++++++++ deploy/thothii.env.example | 22 ++++++++++ docker-compose.dev.yml | 51 ++++++++++++++++++++++++ docker/core-entrypoint.sh | 27 +++++++++++++ docker/core.Dockerfile | 66 ++++++++++++++++++++++++++++++ docker/frontend.Dockerfile | 21 ++++++++++ docker/nginx.conf | 36 +++++++++++++++++ harness/workspaces/local.yaml | 73 ++++++++++++++++++++++++++++++++++ scripts/docker-smoke.sh | 35 ++++++++++++++++ 13 files changed, 436 insertions(+) create mode 100644 .dockerignore create mode 100644 compose.yaml create mode 100644 deploy/sql/10-dwh-roles.sql create mode 100644 deploy/sql/20-vector-roles.sql create mode 100644 deploy/thothii.env.example create mode 100644 docker-compose.dev.yml create mode 100755 docker/core-entrypoint.sh create mode 100644 docker/core.Dockerfile create mode 100644 docker/frontend.Dockerfile create mode 100644 docker/nginx.conf create mode 100644 harness/workspaces/local.yaml create mode 100755 scripts/docker-smoke.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..22d546d3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# build artefacts & deps +**/node_modules +**/.venv +**/__pycache__ +**/.pytest_cache +**/dist +**/*.pyc +harness/.env +harness/workspaces/psd.yaml +deploy/thothii.env +.git +.gitignore +**/*.log +**/.DS_Store +tht-workspace-psd +# docs/site (mkdocs build) — non necessari nelle immagini +docs/superpowers/plans diff --git a/.gitignore b/.gitignore index 4b77a5b3..59867390 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ tools/replay/web/ # === Secrets — NEVER commit === .env +harness/.env +deploy/thothii.env *.pem ca-chain.pem config/ca-chain.pem diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..70fcde5e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,48 @@ +# ThothII — deploy embedded nel portale omics_portal (PRODUZIONE). +# core + frontend sulla rete esterna omics_network (creata dal compose del portale). +# NESSUNA porta host esposta: il backend è invisibile dall'esterno; il portale proxya via service name. +# +# Prereq: il portale deve essere up (crea omics_network): +# cd /home/chirone/omics_portal && docker compose up -d +# Poi: docker compose up -d --build +name: thothii + +services: + core: + build: + context: . + dockerfile: docker/core.Dockerfile + image: thothii-core:local + env_file: [deploy/thothii.env] + environment: + HOST: 0.0.0.0 + PORT: "8787" + THT_HARNESS_DIR: /app/harness + THT_BIN: /opt/venv/bin/tht + PI_BIN: pi + AUTH_MODE: ${AUTH_MODE:-none} + SETTINGS_FILE: /data/settings/settings.json + MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4} + extra_hosts: + - "host.docker.internal:host-gateway" # Supabase :5438 + Ollama :11434 sull'host + volumes: + - /home/chirone/thothii-data:/data + - /home/chirone/thothii-data/pi-config:/home/thoth/.pi + - /home/chirone/chirone/etl/docs/evidence:/data/evidence:ro + restart: unless-stopped + networks: [omics_network] + + frontend: + build: + context: . + dockerfile: docker/frontend.Dockerfile + args: + VITE_BASE: /datamart-builder/assets/ + VITE_BACKEND_URL: /datamart-builder/api + image: thothii-frontend:local + restart: unless-stopped + networks: [omics_network] + +networks: + omics_network: + external: true diff --git a/deploy/sql/10-dwh-roles.sql b/deploy/sql/10-dwh-roles.sql new file mode 100644 index 00000000..3d27573b --- /dev/null +++ b/deploy/sql/10-dwh-roles.sql @@ -0,0 +1,15 @@ +-- ThothII — ruolo DWH read-only (schema datawarehouse). +-- Eseguire sulla stessa istanza Postgres usata da ThothII (porta 5438, accesso diretto). +-- Sostituire :PWD con un secret forte al momento dell'esecuzione: +-- psql -h localhost -p 5438 -U postgres -d postgres -v PWD='' -f 10-dwh-roles.sql +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'thoth_dwh_reader') THEN + CREATE ROLE thoth_dwh_reader LOGIN PASSWORD :'PWD'; + END IF; +END $$; + +GRANT USAGE ON SCHEMA datawarehouse TO thoth_dwh_reader; +GRANT SELECT ON ALL TABLES IN SCHEMA datawarehouse TO thoth_dwh_reader; +ALTER DEFAULT PRIVILEGES IN SCHEMA datawarehouse + GRANT SELECT ON TABLES TO thoth_dwh_reader; diff --git a/deploy/sql/20-vector-roles.sql b/deploy/sql/20-vector-roles.sql new file mode 100644 index 00000000..1753db7f --- /dev/null +++ b/deploy/sql/20-vector-roles.sql @@ -0,0 +1,23 @@ +-- ThothII — ruolo vector read+write (schema vectors). +-- Stessa istanza del DWH (porta 5438). ThothII indicizza (write) + ricerca (read) direttamente. +-- La separazione reader/writer resta rilevante solo per il path REST (non usato in Profile A). +-- psql -h localhost -p 5438 -U postgres -d postgres -v PWD='' -f 20-vector-roles.sql +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'thoth_vector_rw') THEN + CREATE ROLE thoth_vector_rw LOGIN PASSWORD :'PWD'; + END IF; +END $$; + +CREATE SCHEMA IF NOT EXISTS vectors; + +-- L'estensione pgvector deve esistere (già presente nell'istanza di produzione). +-- CREATE EXTENSION IF NOT EXISTS vector; + +GRANT USAGE, CREATE ON SCHEMA vectors TO thoth_vector_rw; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA vectors TO thoth_vector_rw; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA vectors TO thoth_vector_rw; +ALTER DEFAULT PRIVILEGES IN SCHEMA vectors + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO thoth_vector_rw; +ALTER DEFAULT PRIVILEGES IN SCHEMA vectors + GRANT USAGE, SELECT ON SEQUENCES TO thoth_vector_rw; diff --git a/deploy/thothii.env.example b/deploy/thothii.env.example new file mode 100644 index 00000000..2abcfeba --- /dev/null +++ b/deploy/thothii.env.example @@ -0,0 +1,22 @@ +# ThothII core — env di runtime (compose env_file). +# Copiare in deploy/thothii.env e completare. NON committare thothii.env. + +# --- DWH (direct, ruolo read-only su schema datawarehouse) --- +THT_DB_HOST=host.docker.internal +THT_DB_PORT=5438 +THT_DB_NAME=postgres +THT_DB_USER=thoth_dwh_reader +THT_DB_PASSWORD=__CHANGE_ME__ + +# --- Vector (direct, ruolo read+write su schema vectors; stessa istanza del DWH) --- +THT_VEC_HOST=host.docker.internal +THT_VEC_PORT=5438 +THT_VEC_USER=thoth_vector_rw +THT_VEC_PASSWORD=__CHANGE_ME__ + +# --- Embeddings (Ollama sull'host, modello nomic-embed-text-v2-moe) --- +THT_OLLAMA_URL=http://host.docker.internal:11434 + +# --- Backend --- +AUTH_MODE=none # none | mock | oidc (in embedded l'auth è al bordo del portale) +MAX_PI_PROCESSES=4 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..9ac0c780 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,51 @@ +# ThothII — deploy STANDALONE locale (dev / smoke test, senza portale). +# Rete propria + porte host per ispezione diretta. +# docker compose -f docker-compose.dev.yml up -d --build +# frontend: http://localhost:8090 backend: http://localhost:8787 +name: thothii-dev + +services: + core: + build: + context: . + dockerfile: docker/core.Dockerfile + image: thothii-core:local + env_file: [deploy/thothii.env] + environment: + HOST: 0.0.0.0 + PORT: "8787" + THT_HARNESS_DIR: /app/harness + THT_BIN: /opt/venv/bin/tht + PI_BIN: pi + AUTH_MODE: ${AUTH_MODE:-none} + SETTINGS_FILE: /data/settings/settings.json + MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4} + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - /home/chirone/thothii-data:/data + - /home/chirone/thothii-data/pi-config:/home/thoth/.pi + - /home/chirone/chirone/etl/docs/evidence:/data/evidence:ro + ports: + - "8787:8787" + restart: "no" + networks: [thothii-net] + + frontend: + build: + context: . + dockerfile: docker/frontend.Dockerfile + args: + VITE_BASE: / + VITE_BACKEND_URL: /api + image: thothii-frontend:local + ports: + - "8090:8080" + depends_on: + - core + restart: "no" + networks: [thothii-net] + +networks: + thothii-net: + driver: bridge diff --git a/docker/core-entrypoint.sh b/docker/core-entrypoint.sh new file mode 100755 index 00000000..cdf3bbe9 --- /dev/null +++ b/docker/core-entrypoint.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Entrypoints logici del container thothii-core: +# server (default) | check | tht | preprocess +# THT_CONFIG punta al workspace attivo (local.yaml nel deploy co-locato). +set -euo pipefail +export THT_CONFIG="${THT_CONFIG:-/app/harness/workspaces/local.yaml}" + +cmd="${1:-server}" +case "$cmd" in + check) + # Diagnostica di wiring: validazione config/env + ping DWH (read-only). + shift + tht config check -c "$THT_CONFIG" + tht db ping -c "$THT_CONFIG" || echo "(db ping non verde: verificare .env/ruoli/VPN)" + ;; + tht) + shift + exec tht "$@" + ;; + preprocess) + shift + exec tht evidence index "$@" + ;; + *) + exec "$@" + ;; +esac diff --git a/docker/core.Dockerfile b/docker/core.Dockerfile new file mode 100644 index 00000000..0015857b --- /dev/null +++ b/docker/core.Dockerfile @@ -0,0 +1,66 @@ +# syntax=docker/dockerfile:1.7 +# thothii-core: Fastify (Node 22) + harness Python 3.12 (tht CLI) + runtime Pi. +# Singolo container, entrypoint logico "server" (default). +ARG PI_VERSION=0.80.2 + +# ---- Stage 1: backend TypeScript -> dist ---- +FROM node:22-bookworm AS backend-build +WORKDIR /src/backend +COPY backend/package*.json ./ +RUN npm ci +COPY backend/ ./ +RUN npm run build + +# ---- Stage 2: runtime (Python 3.12 nativo + Node 22 copiato, stesso glibc bookworm) ---- +FROM python:3.12-slim-bookworm AS runtime +ARG PI_VERSION + +# Runtime tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates ripgrep fd-find tini \ + && rm -rf /var/lib/apt/lists/* \ + && ln -s /usr/bin/fdfind /usr/local/bin/fd + +# Node 22 + npm copiati dall'immagine ufficiale (stesso Debian bookworm → binario compatibile) +COPY --from=node:22-bookworm /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22-bookworm /usr/local/lib/node_modules /usr/local/lib/node_modules +RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ + && ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx + +# Utente non-root +RUN useradd --create-home --uid 10001 --shell /bin/bash thoth + +# Harness: venv nativo (python 3.12) + tht installato NON editabile (nel venv, indipendente dal path sorgente). +# psycopg2-binary è wheel → niente gcc/libpq-dev. yake/sqlglot/datasketch/pydantic hanno wheel per py3.12. +COPY harness/ /app/harness/ +RUN python -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir --upgrade pip \ + && /opt/venv/bin/pip install --no-cache-dir /app/harness +# PiProcessManager (backend) prepende harnessDir/.venv/bin al PATH del child Pi → symlink al venv reale +RUN ln -s /opt/venv /app/harness/.venv + +# Backend: dist + node_modules (stesso Node major 22 + glibc bookworm → compatibili) +COPY --from=backend-build /src/backend/dist /app/backend/dist +COPY --from=backend-build /src/backend/node_modules /app/backend/node_modules +COPY backend/package*.json /app/backend/ + +# Runtime Pi (pacchetto npm puro JS, dipendenze prebuilt). Installato come root, eseguibile da thoth. +RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION} + +ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH" \ + HOST=0.0.0.0 PORT=8787 \ + THT_HARNESS_DIR=/app/harness \ + THT_BIN=/opt/venv/bin/tht \ + PI_BIN=pi \ + HOME=/home/thoth + +COPY docker/core-entrypoint.sh /app/docker/core-entrypoint.sh +RUN chmod +x /app/docker/core-entrypoint.sh + +WORKDIR /app/backend +USER thoth +EXPOSE 8787 +HEALTHCHECK --interval=15s --timeout=3s --retries=5 --start-period=30s \ + CMD curl -fsS http://127.0.0.1:8787/health || exit 1 +ENTRYPOINT ["/usr/bin/tini","--","/app/docker/core-entrypoint.sh"] +CMD ["server"] diff --git a/docker/frontend.Dockerfile b/docker/frontend.Dockerfile new file mode 100644 index 00000000..f3a09b71 --- /dev/null +++ b/docker/frontend.Dockerfile @@ -0,0 +1,21 @@ +# syntax=docker/dockerfile:1.7 +# thothii-frontend: build Vite (React) + nginx-unprivileged (porta 8080). +# Build args: +# VITE_BASE prefisso asset ("/" standalone, "/datamart-builder/assets/" embedded) +# VITE_BACKEND_URL base API ("http://localhost:8787" standalone, "/datamart-builder/api" embedded) +FROM node:22-bookworm AS build +WORKDIR /src +COPY frontend/package*.json ./ +RUN npm ci +COPY frontend/ ./ +ARG VITE_BASE=/ +ARG VITE_BACKEND_URL=http://localhost:8787 +ENV VITE_BASE=$VITE_BASE VITE_BACKEND_URL=$VITE_BACKEND_URL +RUN npm run build +# typecheck opzionale (non bloccante nella build dell'immagine) +RUN npx tsc -b 2>/dev/null || true + +FROM nginxinc/nginx-unprivileged:1.27-alpine AS runtime +COPY --from=build /src/dist /usr/share/nginx/html +COPY docker/nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 8080 diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 00000000..dcb83245 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,36 @@ +# nginx per thothii-frontend: serve la SPA (modalità standalone) e reverse-proxy /api -> core. +# In modalità embedded il portale proxya /datamart-builder/assets/ qui (solo asset statici); +# il blocco /api non è usato in embedded (il portale hita core direttamente). +server { + listen 8080; + server_name _; + root /usr/share/nginx/html; + index index.html; + + # SPA fallback (standalone) + location / { + try_files $uri $uri/ /index.html; + } + + # Reverse proxy verso il backend (stessa rete Docker) + location /api/ { + proxy_pass http://core:8787/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # SSE: niente buffering, timeout lunghi + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + chunked_transfer_encoding on; + } + + # manifest.json servito (lo legge il template tag Django in embedded) + location = /manifest.json { + default_type application/json; + } +} diff --git a/harness/workspaces/local.yaml b/harness/workspaces/local.yaml new file mode 100644 index 00000000..f763db39 --- /dev/null +++ b/harness/workspaces/local.yaml @@ -0,0 +1,73 @@ +# Workspace ThothII — Profilo A (server co-locato). DWH + vector BOTH direct, no REST. +# Segreti SOLO in env (compose env_file: deploy/thothii.env). Path assoluti interni al container (/data). +language: it + +database: + host: ${THT_DB_HOST} # host.docker.internal + port: ${THT_DB_PORT} # 5438 (stessa istanza del vector) + database: ${THT_DB_NAME} # postgres + schema: datawarehouse + user: ${THT_DB_USER} # thoth_dwh_reader + password: ${THT_DB_PASSWORD} + transport: direct # Postgres diretto, niente PostgREST/CA + +# Nessuna sezione `rest`: non usata con transport=direct. + +paths: + sessions: /data/sessions + artifacts: /data/artifacts + indexes: /data/indexes + +examples: + max_per_column: 10 + +lsh: + signature_size: 64 + n_gram: 3 + threshold: 0.5 + max_values_per_column: 1000 + +eligibility: + max_declared_len: 128 + max_avg_length: 40 + max_sampled_len: 200 + ignore_columns: [etl_last_update] + +evidence: + source_root: /data # il corpus è montato a /data/evidence + evidence_dir: evidence # → /data/evidence + +embeddings: + base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434 + model: nomic-embed-text-v2-moe + dim: 768 + batch_size: 32 + +# Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che +# open_searcher()/open_store() (harness/tht/cli/vector_cmd.py) selezionino DirectSearcher/VectorStore. +vector_db: + host: ${THT_VEC_HOST} # host.docker.internal + port: ${THT_VEC_PORT} # 5438 + database: postgres + schema: vectors + user: ${THT_VEC_USER} # thoth_vector_rw + password: ${THT_VEC_PASSWORD} + +# Nessuna sezione vector_rest / vector_write_rest: tutto diretto in locale. + +vector: + max_chunk_chars: 4000 + +search: + rrf_k: 60 + top_schema_tables: 12 + schema_chunk_pool: 150 + +execution: + allow: [cte_test, explain, preview, aggregate, export] + max_preview_rows: 10 + max_export_rows: 100000 + statement_timeout_ms: 30000 + warn_execution_ms: 5000 + max_aggregate_cells: 20 + forbidden_functions: [set_config, dblink, dblink_exec, lo_import] diff --git a/scripts/docker-smoke.sh b/scripts/docker-smoke.sh new file mode 100755 index 00000000..cfe357ca --- /dev/null +++ b/scripts/docker-smoke.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Smoke test del deploy standalone ThothII (core + frontend). +# Usa docker-compose.dev.yml (rete propria, porte host). Non tocca il portale. +# Prereq: deploy/thothii.env popolato + ruoli DB creati + pi-config + settings.json. +set -euo pipefail +cd "$(dirname "$0")/.." + +DC="docker compose -f docker-compose.dev.yml" +WS="/app/harness/workspaces/local.yaml" + +echo "== ThothII standalone smoke ==" +$DC config --quiet + +echo "== Build ==" +$DC build + +echo "== Up (wait health) ==" +$DC up -d --wait + +echo "== Core health ==" +curl -fsS http://localhost:8787/health && echo + +echo "== Frontend serve ==" +curl -fsSI http://localhost:8090/ | head -1 + +echo "== Wiring check (config + DWH ping; -c è per-command) ==" +$DC exec -T core tht config check -c "$WS" || \ + echo "(config check non verde: verificare .env/ruoli DB)" +$DC exec -T core tht db ping -c "$WS" || \ + echo "(db ping non verde: verificare ruolo thoth_dwh_reader + rete)" + +echo "== Down ==" +$DC down + +echo "OK: smoke standalone passato."