feat(deploy): docker images, compose, roles SQL, local workspace
- core.Dockerfile: python:3.12-slim + node 22 copied (same bookworm glibc), non-root, tht+pi
- frontend.Dockerfile: vite build (env-driven base/assetsDir) + nginx-unprivileged
- compose.yaml (embedded, omics_network ext, zero host ports) + docker-compose.dev.yml (standalone)
- deploy/sql: thoth_dwh_reader (ro) + thoth_vector_rw (rw) roles
- deploy/thothii.env.example + harness/workspaces/local.yaml (direct DWH+vector, 5438)
- scripts/docker-smoke.sh; .dockerignore; gitignore deploy secrets
- verified: both images build, core health {ok}, config check validates local.yaml
This commit is contained in:
User
@@ -0,0 +1,17 @@
|
||||
# build artefacts & deps
|
||||
**/node_modules
|
||||
**/.venv
|
||||
**/__pycache__
|
||||
**/.pytest_cache
|
||||
**/dist
|
||||
**/*.pyc
|
||||
harness/.env
|
||||
harness/workspaces/psd.yaml
|
||||
deploy/thothii.env
|
||||
.git
|
||||
.gitignore
|
||||
**/*.log
|
||||
**/.DS_Store
|
||||
tht-workspace-psd
|
||||
# docs/site (mkdocs build) — non necessari nelle immagini
|
||||
docs/superpowers/plans
|
||||
@@ -27,6 +27,8 @@ tools/replay/web/
|
||||
|
||||
# === Secrets — NEVER commit ===
|
||||
.env
|
||||
harness/.env
|
||||
deploy/thothii.env
|
||||
*.pem
|
||||
ca-chain.pem
|
||||
config/ca-chain.pem
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# ThothII — deploy embedded nel portale omics_portal (PRODUZIONE).
|
||||
# core + frontend sulla rete esterna omics_network (creata dal compose del portale).
|
||||
# NESSUNA porta host esposta: il backend è invisibile dall'esterno; il portale proxya via service name.
|
||||
#
|
||||
# Prereq: il portale deve essere up (crea omics_network):
|
||||
# cd /home/chirone/omics_portal && docker compose up -d
|
||||
# Poi: docker compose up -d --build
|
||||
name: thothii
|
||||
|
||||
services:
|
||||
core:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/core.Dockerfile
|
||||
image: thothii-core:local
|
||||
env_file: [deploy/thothii.env]
|
||||
environment:
|
||||
HOST: 0.0.0.0
|
||||
PORT: "8787"
|
||||
THT_HARNESS_DIR: /app/harness
|
||||
THT_BIN: /opt/venv/bin/tht
|
||||
PI_BIN: pi
|
||||
AUTH_MODE: ${AUTH_MODE:-none}
|
||||
SETTINGS_FILE: /data/settings/settings.json
|
||||
MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway" # Supabase :5438 + Ollama :11434 sull'host
|
||||
volumes:
|
||||
- /home/chirone/thothii-data:/data
|
||||
- /home/chirone/thothii-data/pi-config:/home/thoth/.pi
|
||||
- /home/chirone/chirone/etl/docs/evidence:/data/evidence:ro
|
||||
restart: unless-stopped
|
||||
networks: [omics_network]
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/frontend.Dockerfile
|
||||
args:
|
||||
VITE_BASE: /datamart-builder/assets/
|
||||
VITE_BACKEND_URL: /datamart-builder/api
|
||||
image: thothii-frontend:local
|
||||
restart: unless-stopped
|
||||
networks: [omics_network]
|
||||
|
||||
networks:
|
||||
omics_network:
|
||||
external: true
|
||||
@@ -0,0 +1,15 @@
|
||||
-- ThothII — ruolo DWH read-only (schema datawarehouse).
|
||||
-- Eseguire sulla stessa istanza Postgres usata da ThothII (porta 5438, accesso diretto).
|
||||
-- Sostituire :PWD con un secret forte al momento dell'esecuzione:
|
||||
-- psql -h localhost -p 5438 -U postgres -d postgres -v PWD='<secret>' -f 10-dwh-roles.sql
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'thoth_dwh_reader') THEN
|
||||
CREATE ROLE thoth_dwh_reader LOGIN PASSWORD :'PWD';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
GRANT USAGE ON SCHEMA datawarehouse TO thoth_dwh_reader;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA datawarehouse TO thoth_dwh_reader;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA datawarehouse
|
||||
GRANT SELECT ON TABLES TO thoth_dwh_reader;
|
||||
@@ -0,0 +1,23 @@
|
||||
-- ThothII — ruolo vector read+write (schema vectors).
|
||||
-- Stessa istanza del DWH (porta 5438). ThothII indicizza (write) + ricerca (read) direttamente.
|
||||
-- La separazione reader/writer resta rilevante solo per il path REST (non usato in Profile A).
|
||||
-- psql -h localhost -p 5438 -U postgres -d postgres -v PWD='<secret>' -f 20-vector-roles.sql
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'thoth_vector_rw') THEN
|
||||
CREATE ROLE thoth_vector_rw LOGIN PASSWORD :'PWD';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS vectors;
|
||||
|
||||
-- L'estensione pgvector deve esistere (già presente nell'istanza di produzione).
|
||||
-- CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
GRANT USAGE, CREATE ON SCHEMA vectors TO thoth_vector_rw;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA vectors TO thoth_vector_rw;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA vectors TO thoth_vector_rw;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA vectors
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO thoth_vector_rw;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA vectors
|
||||
GRANT USAGE, SELECT ON SEQUENCES TO thoth_vector_rw;
|
||||
@@ -0,0 +1,22 @@
|
||||
# ThothII core — env di runtime (compose env_file).
|
||||
# Copiare in deploy/thothii.env e completare. NON committare thothii.env.
|
||||
|
||||
# --- DWH (direct, ruolo read-only su schema datawarehouse) ---
|
||||
THT_DB_HOST=host.docker.internal
|
||||
THT_DB_PORT=5438
|
||||
THT_DB_NAME=postgres
|
||||
THT_DB_USER=thoth_dwh_reader
|
||||
THT_DB_PASSWORD=__CHANGE_ME__
|
||||
|
||||
# --- Vector (direct, ruolo read+write su schema vectors; stessa istanza del DWH) ---
|
||||
THT_VEC_HOST=host.docker.internal
|
||||
THT_VEC_PORT=5438
|
||||
THT_VEC_USER=thoth_vector_rw
|
||||
THT_VEC_PASSWORD=__CHANGE_ME__
|
||||
|
||||
# --- Embeddings (Ollama sull'host, modello nomic-embed-text-v2-moe) ---
|
||||
THT_OLLAMA_URL=http://host.docker.internal:11434
|
||||
|
||||
# --- Backend ---
|
||||
AUTH_MODE=none # none | mock | oidc (in embedded l'auth è al bordo del portale)
|
||||
MAX_PI_PROCESSES=4
|
||||
@@ -0,0 +1,51 @@
|
||||
# ThothII — deploy STANDALONE locale (dev / smoke test, senza portale).
|
||||
# Rete propria + porte host per ispezione diretta.
|
||||
# docker compose -f docker-compose.dev.yml up -d --build
|
||||
# frontend: http://localhost:8090 backend: http://localhost:8787
|
||||
name: thothii-dev
|
||||
|
||||
services:
|
||||
core:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/core.Dockerfile
|
||||
image: thothii-core:local
|
||||
env_file: [deploy/thothii.env]
|
||||
environment:
|
||||
HOST: 0.0.0.0
|
||||
PORT: "8787"
|
||||
THT_HARNESS_DIR: /app/harness
|
||||
THT_BIN: /opt/venv/bin/tht
|
||||
PI_BIN: pi
|
||||
AUTH_MODE: ${AUTH_MODE:-none}
|
||||
SETTINGS_FILE: /data/settings/settings.json
|
||||
MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
- /home/chirone/thothii-data:/data
|
||||
- /home/chirone/thothii-data/pi-config:/home/thoth/.pi
|
||||
- /home/chirone/chirone/etl/docs/evidence:/data/evidence:ro
|
||||
ports:
|
||||
- "8787:8787"
|
||||
restart: "no"
|
||||
networks: [thothii-net]
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/frontend.Dockerfile
|
||||
args:
|
||||
VITE_BASE: /
|
||||
VITE_BACKEND_URL: /api
|
||||
image: thothii-frontend:local
|
||||
ports:
|
||||
- "8090:8080"
|
||||
depends_on:
|
||||
- core
|
||||
restart: "no"
|
||||
networks: [thothii-net]
|
||||
|
||||
networks:
|
||||
thothii-net:
|
||||
driver: bridge
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# Entrypoints logici del container thothii-core:
|
||||
# server (default) | check | tht <args...> | preprocess <args...>
|
||||
# THT_CONFIG punta al workspace attivo (local.yaml nel deploy co-locato).
|
||||
set -euo pipefail
|
||||
export THT_CONFIG="${THT_CONFIG:-/app/harness/workspaces/local.yaml}"
|
||||
|
||||
cmd="${1:-server}"
|
||||
case "$cmd" in
|
||||
check)
|
||||
# Diagnostica di wiring: validazione config/env + ping DWH (read-only).
|
||||
shift
|
||||
tht config check -c "$THT_CONFIG"
|
||||
tht db ping -c "$THT_CONFIG" || echo "(db ping non verde: verificare .env/ruoli/VPN)"
|
||||
;;
|
||||
tht)
|
||||
shift
|
||||
exec tht "$@"
|
||||
;;
|
||||
preprocess)
|
||||
shift
|
||||
exec tht evidence index "$@"
|
||||
;;
|
||||
*)
|
||||
exec "$@"
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,66 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# thothii-core: Fastify (Node 22) + harness Python 3.12 (tht CLI) + runtime Pi.
|
||||
# Singolo container, entrypoint logico "server" (default).
|
||||
ARG PI_VERSION=0.80.2
|
||||
|
||||
# ---- Stage 1: backend TypeScript -> dist ----
|
||||
FROM node:22-bookworm AS backend-build
|
||||
WORKDIR /src/backend
|
||||
COPY backend/package*.json ./
|
||||
RUN npm ci
|
||||
COPY backend/ ./
|
||||
RUN npm run build
|
||||
|
||||
# ---- Stage 2: runtime (Python 3.12 nativo + Node 22 copiato, stesso glibc bookworm) ----
|
||||
FROM python:3.12-slim-bookworm AS runtime
|
||||
ARG PI_VERSION
|
||||
|
||||
# Runtime tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl ca-certificates ripgrep fd-find tini \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -s /usr/bin/fdfind /usr/local/bin/fd
|
||||
|
||||
# Node 22 + npm copiati dall'immagine ufficiale (stesso Debian bookworm → binario compatibile)
|
||||
COPY --from=node:22-bookworm /usr/local/bin/node /usr/local/bin/node
|
||||
COPY --from=node:22-bookworm /usr/local/lib/node_modules /usr/local/lib/node_modules
|
||||
RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
|
||||
&& ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx
|
||||
|
||||
# Utente non-root
|
||||
RUN useradd --create-home --uid 10001 --shell /bin/bash thoth
|
||||
|
||||
# Harness: venv nativo (python 3.12) + tht installato NON editabile (nel venv, indipendente dal path sorgente).
|
||||
# psycopg2-binary è wheel → niente gcc/libpq-dev. yake/sqlglot/datasketch/pydantic hanno wheel per py3.12.
|
||||
COPY harness/ /app/harness/
|
||||
RUN python -m venv /opt/venv \
|
||||
&& /opt/venv/bin/pip install --no-cache-dir --upgrade pip \
|
||||
&& /opt/venv/bin/pip install --no-cache-dir /app/harness
|
||||
# PiProcessManager (backend) prepende harnessDir/.venv/bin al PATH del child Pi → symlink al venv reale
|
||||
RUN ln -s /opt/venv /app/harness/.venv
|
||||
|
||||
# Backend: dist + node_modules (stesso Node major 22 + glibc bookworm → compatibili)
|
||||
COPY --from=backend-build /src/backend/dist /app/backend/dist
|
||||
COPY --from=backend-build /src/backend/node_modules /app/backend/node_modules
|
||||
COPY backend/package*.json /app/backend/
|
||||
|
||||
# Runtime Pi (pacchetto npm puro JS, dipendenze prebuilt). Installato come root, eseguibile da thoth.
|
||||
RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}
|
||||
|
||||
ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH" \
|
||||
HOST=0.0.0.0 PORT=8787 \
|
||||
THT_HARNESS_DIR=/app/harness \
|
||||
THT_BIN=/opt/venv/bin/tht \
|
||||
PI_BIN=pi \
|
||||
HOME=/home/thoth
|
||||
|
||||
COPY docker/core-entrypoint.sh /app/docker/core-entrypoint.sh
|
||||
RUN chmod +x /app/docker/core-entrypoint.sh
|
||||
|
||||
WORKDIR /app/backend
|
||||
USER thoth
|
||||
EXPOSE 8787
|
||||
HEALTHCHECK --interval=15s --timeout=3s --retries=5 --start-period=30s \
|
||||
CMD curl -fsS http://127.0.0.1:8787/health || exit 1
|
||||
ENTRYPOINT ["/usr/bin/tini","--","/app/docker/core-entrypoint.sh"]
|
||||
CMD ["server"]
|
||||
@@ -0,0 +1,21 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# thothii-frontend: build Vite (React) + nginx-unprivileged (porta 8080).
|
||||
# Build args:
|
||||
# VITE_BASE prefisso asset ("/" standalone, "/datamart-builder/assets/" embedded)
|
||||
# VITE_BACKEND_URL base API ("http://localhost:8787" standalone, "/datamart-builder/api" embedded)
|
||||
FROM node:22-bookworm AS build
|
||||
WORKDIR /src
|
||||
COPY frontend/package*.json ./
|
||||
RUN npm ci
|
||||
COPY frontend/ ./
|
||||
ARG VITE_BASE=/
|
||||
ARG VITE_BACKEND_URL=http://localhost:8787
|
||||
ENV VITE_BASE=$VITE_BASE VITE_BACKEND_URL=$VITE_BACKEND_URL
|
||||
RUN npm run build
|
||||
# typecheck opzionale (non bloccante nella build dell'immagine)
|
||||
RUN npx tsc -b 2>/dev/null || true
|
||||
|
||||
FROM nginxinc/nginx-unprivileged:1.27-alpine AS runtime
|
||||
COPY --from=build /src/dist /usr/share/nginx/html
|
||||
COPY docker/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
EXPOSE 8080
|
||||
@@ -0,0 +1,36 @@
|
||||
# nginx per thothii-frontend: serve la SPA (modalità standalone) e reverse-proxy /api -> core.
|
||||
# In modalità embedded il portale proxya /datamart-builder/assets/ qui (solo asset statici);
|
||||
# il blocco /api non è usato in embedded (il portale hita core direttamente).
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# SPA fallback (standalone)
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# Reverse proxy verso il backend (stessa rete Docker)
|
||||
location /api/ {
|
||||
proxy_pass http://core:8787/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# SSE: niente buffering, timeout lunghi
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_read_timeout 86400s;
|
||||
proxy_send_timeout 86400s;
|
||||
chunked_transfer_encoding on;
|
||||
}
|
||||
|
||||
# manifest.json servito (lo legge il template tag Django in embedded)
|
||||
location = /manifest.json {
|
||||
default_type application/json;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
# Workspace ThothII — Profilo A (server co-locato). DWH + vector BOTH direct, no REST.
|
||||
# Segreti SOLO in env (compose env_file: deploy/thothii.env). Path assoluti interni al container (/data).
|
||||
language: it
|
||||
|
||||
database:
|
||||
host: ${THT_DB_HOST} # host.docker.internal
|
||||
port: ${THT_DB_PORT} # 5438 (stessa istanza del vector)
|
||||
database: ${THT_DB_NAME} # postgres
|
||||
schema: datawarehouse
|
||||
user: ${THT_DB_USER} # thoth_dwh_reader
|
||||
password: ${THT_DB_PASSWORD}
|
||||
transport: direct # Postgres diretto, niente PostgREST/CA
|
||||
|
||||
# Nessuna sezione `rest`: non usata con transport=direct.
|
||||
|
||||
paths:
|
||||
sessions: /data/sessions
|
||||
artifacts: /data/artifacts
|
||||
indexes: /data/indexes
|
||||
|
||||
examples:
|
||||
max_per_column: 10
|
||||
|
||||
lsh:
|
||||
signature_size: 64
|
||||
n_gram: 3
|
||||
threshold: 0.5
|
||||
max_values_per_column: 1000
|
||||
|
||||
eligibility:
|
||||
max_declared_len: 128
|
||||
max_avg_length: 40
|
||||
max_sampled_len: 200
|
||||
ignore_columns: [etl_last_update]
|
||||
|
||||
evidence:
|
||||
source_root: /data # il corpus è montato a /data/evidence
|
||||
evidence_dir: evidence # → /data/evidence
|
||||
|
||||
embeddings:
|
||||
base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434
|
||||
model: nomic-embed-text-v2-moe
|
||||
dim: 768
|
||||
batch_size: 32
|
||||
|
||||
# Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che
|
||||
# open_searcher()/open_store() (harness/tht/cli/vector_cmd.py) selezionino DirectSearcher/VectorStore.
|
||||
vector_db:
|
||||
host: ${THT_VEC_HOST} # host.docker.internal
|
||||
port: ${THT_VEC_PORT} # 5438
|
||||
database: postgres
|
||||
schema: vectors
|
||||
user: ${THT_VEC_USER} # thoth_vector_rw
|
||||
password: ${THT_VEC_PASSWORD}
|
||||
|
||||
# Nessuna sezione vector_rest / vector_write_rest: tutto diretto in locale.
|
||||
|
||||
vector:
|
||||
max_chunk_chars: 4000
|
||||
|
||||
search:
|
||||
rrf_k: 60
|
||||
top_schema_tables: 12
|
||||
schema_chunk_pool: 150
|
||||
|
||||
execution:
|
||||
allow: [cte_test, explain, preview, aggregate, export]
|
||||
max_preview_rows: 10
|
||||
max_export_rows: 100000
|
||||
statement_timeout_ms: 30000
|
||||
warn_execution_ms: 5000
|
||||
max_aggregate_cells: 20
|
||||
forbidden_functions: [set_config, dblink, dblink_exec, lo_import]
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# Smoke test del deploy standalone ThothII (core + frontend).
|
||||
# Usa docker-compose.dev.yml (rete propria, porte host). Non tocca il portale.
|
||||
# Prereq: deploy/thothii.env popolato + ruoli DB creati + pi-config + settings.json.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
DC="docker compose -f docker-compose.dev.yml"
|
||||
WS="/app/harness/workspaces/local.yaml"
|
||||
|
||||
echo "== ThothII standalone smoke =="
|
||||
$DC config --quiet
|
||||
|
||||
echo "== Build =="
|
||||
$DC build
|
||||
|
||||
echo "== Up (wait health) =="
|
||||
$DC up -d --wait
|
||||
|
||||
echo "== Core health =="
|
||||
curl -fsS http://localhost:8787/health && echo
|
||||
|
||||
echo "== Frontend serve =="
|
||||
curl -fsSI http://localhost:8090/ | head -1
|
||||
|
||||
echo "== Wiring check (config + DWH ping; -c è per-command) =="
|
||||
$DC exec -T core tht config check -c "$WS" || \
|
||||
echo "(config check non verde: verificare .env/ruoli DB)"
|
||||
$DC exec -T core tht db ping -c "$WS" || \
|
||||
echo "(db ping non verde: verificare ruolo thoth_dwh_reader + rete)"
|
||||
|
||||
echo "== Down =="
|
||||
$DC down
|
||||
|
||||
echo "OK: smoke standalone passato."
|
||||
Reference in New Issue
Block a user