diff --git a/.superpowers/sdd/task-4-report.md b/.superpowers/sdd/task-4-report.md new file mode 100644 index 00000000..119100f9 --- /dev/null +++ b/.superpowers/sdd/task-4-report.md @@ -0,0 +1,45 @@ +# Task 4 report — one-command Docker documentation + +## Status + +Implemented. The installation documentation now uses the canonical flow: + +```sh +cp .env.example .env +cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets +chmod 600 deploy/secrets/thothii.secrets +docker compose up --build -d +``` + +Updated: + +- `README.md` with root `.env` defaults, one bundle, optional overlay presets, CA limitation, + preprocessing, and migration notes. +- `docs/installazione-docker-4-contesti.md` rewritten with exact files to create/edit and the + four requested contexts (co-located DB/vector, Mac, Windows, and remote DB/Evidence server). +- `docs/index.md` link text for the one-command installation. +- `deploy/secrets/README.md` bundle syntax, permissions, runtime mount verification, CA handling, + and migration guidance. +- `scripts/docker-smoke.sh` now creates a disposable mode-0600 bundle and exercises the default + Compose services without the legacy `external` profile. +- `scripts/test-default-compose.sh` asserts the exact installation command, tracked templates, + and absence of the legacy setup in the guide. + +The docs explicitly state that a PEM CA chain cannot be put in the strict single-line bundle. A +reviewed Compose override/secret-manager mount is required for `THT_SSL_CA`. Direct PostgreSQL +workspace examples are marked as advanced and require a separate reviewed runtime password mount; +the base bundle mount is the only default mount. + +## Verification + +- `sh -n scripts/docker-smoke.sh scripts/test-default-compose.sh` — passed. +- `./scripts/test-default-compose.sh` — passed. +- `git diff --check` — passed. +- `./scripts/test-docker-smoke.sh` — passed after updating its static assertion to the default + no-profile invocation. + +## Concerns + +The legacy `scripts/vector-rotate-bootstrap-password.sh` maintenance helper still accepts +old/new standalone files. Its output is intentionally documented as a transitional interface; +the resulting value must be copied into the bundle before restarting local-vector services. diff --git a/README.md b/README.md index 69d7fbf1..8059d471 100644 --- a/README.md +++ b/README.md @@ -4,24 +4,53 @@ ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fasti core. The portable deployment runs exactly two application services; data services remain external in this profile. -## Docker Compose: external services +## Docker Compose: one-command startup -Requirements: Docker Engine with Compose v2 and reachable DWH, vector, and embeddings -services. +Requirements: Docker Engine with Compose v2. The default project starts only the two +application images; DWH, vector and embedding services can be remote or supplied by an +optional overlay. -1. For local development only, copy `deploy/env.example` to `deploy/.env` and fill in runtime - credentials. The file is gitignored and is never copied into either image. -2. Add or edit YAML workspace descriptors under `deploy/workspaces/`. These files are mounted - read-only. Use relative `roots`; they resolve beneath `/data/workspaces/`. -3. Start the external-service profile: +From a fresh clone, run these commands from the repository root: - ```sh - docker compose -f compose.yaml -f deploy/compose.local.yaml \ - --profile external up --build --wait - ``` +```sh +cp .env.example .env +cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets +chmod 600 deploy/secrets/thothii.secrets +# Edit .env (non-secret endpoints) and deploy/secrets/thothii.secrets (KEY=VALUE lines). +docker compose up --build -d +``` -4. Open . The published port is loopback-only. Set `THOTH_HTTP_PORT` - before starting to use another loopback port. +The root `.env` is loaded automatically by Compose. It defaults to `compose.yaml`, an empty +profile, and `THT_SECRETS_FILE=deploy/secrets/thothii.secrets`; no `--env-file`, `-f`, or +`--profile` flag is required for the normal installation. Add or edit YAML workspace descriptors +under `deploy/workspaces/`; they are mounted read-only and relative `roots` resolve beneath +`/data/workspaces/`. Open (set `THOTH_HTTP_PORT` in +`.env` to choose another loopback port). + +The bundle contains only values, one per line (`THT_MODEL_API_KEY=...`, DWH/vector keys, and +the optional local-vector passwords). It is ignored by Git and never copied into either image. +Do not put credentials in `.env`, workspace YAML, URLs, or Compose interpolation values. + +### Optional overlays + +Overlays are selected in `.env`, so the operational command remains the same. On Unix-like +systems use `:` between files; on Windows use `;`: + +```dotenv +# Remote DWH/vector/embedding services with authenticated reverse proxy: +COMPOSE_FILE=compose.yaml:deploy/compose.production.yaml +COMPOSE_PROFILES= + +# Local pgvector (Mac/Windows or a standalone application server): +COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml +COMPOSE_PROFILES=local-vector +``` + +After changing `.env`, apply the selected configuration with `docker compose up --build -d`. +Preprocessing is an explicit opt-in preset: append +`deploy/compose.preprocess.yaml:deploy/compose.preprocess-local-vector.yaml` and set +`COMPOSE_PROFILES=local-vector,preprocess`; then run the job with +`docker compose run --rm preprocess-evidence` or `preprocess-dwh`. Application state, including settings, sessions, artifacts, and indexes, lives in the named `thoth_data` volume mounted at `/data`. `docker compose down` keeps that volume. Only an @@ -43,41 +72,30 @@ Each run uses a unique Compose project and removes that project's containers, ne volume afterward. It never targets the fixed `thothii` operator project or its volume. Set `SMOKE_PROJECT` to a different explicit project name for reproducible debugging, and set `KEEP_SMOKE_RESOURCES=1` to retain that smoke project's resources for inspection; remove them -later with `docker compose --project-name "$SMOKE_PROJECT" --profile external down --volumes`. +later with `docker compose --project-name "$SMOKE_PROJECT" down --volumes`. ## Optional local pgvector and recovery -Start the persistent local vector profile with `docker compose -f compose.yaml -f -deploy/compose.local-vector.yaml --profile local-vector up --build --wait`. Its `vector_data` -volume is independent of application state. Reader, writer, -migrator, and bootstrap credentials remain separate; password files must be mode `0600` and -must not be passed as URL arguments. +The local-vector overlay reads `THT_VECTOR_BOOTSTRAP_PASSWORD`, +`THT_VECTOR_MIGRATOR_PASSWORD`, `THT_VECTOR_READER_PASSWORD`, and +`THT_VECTOR_WRITER_PASSWORD` from the same bundle. Its `vector_data` volume is independent of +application state; passwords are selected at runtime and are never passed as URL arguments. ## Preprocessing jobs and S3 Evidence -The included job workspaces target the local-vector profile. Point the four -`THT_VECTOR_*_PASSWORD_SECRET_FILE` variables at owner-only files, set `THT_OLLAMA_URL`, mount -Evidence at `/data/source/evidence`, then run the explicit overlays (which are inert for normal -runtime): +The included job workspaces target the local-vector profile. Put the four local-vector password +keys in the bundle, set `THT_OLLAMA_URL`, mount Evidence at `/data/source/evidence`, then select +the preprocessing preset in `.env`: -```sh -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess build preprocess-evidence -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess run --rm preprocess-evidence -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess build preprocess-dwh -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess run --rm preprocess-dwh +```dotenv +COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml:deploy/compose.preprocess.yaml:deploy/compose.preprocess-local-vector.yaml +COMPOSE_PROFILES=local-vector,preprocess ``` -The local preprocessing override makes each job wait for the vector database health check, -role reconciliation, and a successful migration. These commands are safe on a clean Compose -project; no separate database startup or migration command is required. +Run `docker compose run --rm preprocess-evidence` or +`docker compose run --rm preprocess-dwh`. The overlay makes each job wait for the vector +database health check, role reconciliation, and a successful migration; no separate database +startup or migration command is required. S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance. AWS endpoints are used when no custom URL is supplied. Every custom endpoint is an explicit egress @@ -138,36 +156,32 @@ with the organization's reviewed identity proxy. `AUTH_MODE=upstream` trusts thi rejects requests without the identity header. Setting `THOTH_PUBLIC_EXPOSURE=true` with any other auth mode fails during core startup. -Production credentials use Compose secrets, not `deploy/.env`. Create five files outside the -repository, restrict their host permissions, and point these variables to them: +Production credentials use the one Compose secret bundle, not `.env`. Put the required keys in +`deploy/secrets/thothii.secrets` and select the production overlay in `.env`: -```sh -export THT_DWH_API_KEY_SECRET_FILE=/secure/thoth/dwh-api-key -export THT_VEC_API_KEY_SECRET_FILE=/secure/thoth/vector-reader-api-key -export THT_VEC_WRITE_API_KEY_SECRET_FILE=/secure/thoth/vector-writer-api-key -export THT_CA_SECRET_FILE=/secure/thoth/ca-chain.pem -export THT_MODEL_API_KEY_SECRET_FILE=/secure/thoth/model-api-key -export THT_DB_NAME=warehouse -export THT_DWH_REST_URL=https://dwh.example.test -export THT_VEC_REST_URL=https://vectors.example.test -export THT_OLLAMA_URL=https://embeddings.example.test -docker compose -f compose.yaml -f deploy/compose.production.yaml \ - --profile external up --build --wait +```dotenv +THT_MODEL_API_KEY=replace-me +THT_DWH_API_KEY=replace-me +THT_VEC_API_KEY=replace-me +THT_VEC_WRITE_API_KEY=replace-me ``` -The secrets and public CA chain are mounted read-only under `/run/secrets` and must be readable by -the core's UID 10001. Host secret files must be `0600` or `0400`; Docker's runtime `0444` mount is -accepted only beneath `/run/secrets`. See [`deploy/secrets/README.md`](deploy/secrets/README.md) for -the verification command. The frontend remains on loopback; the authenticated host proxy is the +The bundle is mounted read-only as `/run/secrets/thothii.secrets` and must be mode `0600` or +`0400` on the host. Docker's runtime `0444` mode is accepted only beneath `/run/secrets`; see +[`deploy/secrets/README.md`](deploy/secrets/README.md). A PEM CA chain is deliberately not a +bundle value: PEM contains whitespace and is rejected by the strict parser. Keep the CA chain in +the host/secret-manager materialization and add a reviewed Compose override that mounts it at +`/run/secrets/ca-chain.pem` and sets `THT_SSL_CA` when a private CA is required. The base bundle +does not create that mount. The frontend remains on loopback; the authenticated host proxy is the only public listener. Set the selected model provider in application settings (or `PI_PROVIDER`). For each Pi spawn the -backend validates and reads `THT_MODEL_API_KEY_FILE`, then exposes its value only as the provider's +backend validates and reads `THT_MODEL_API_KEY` from the bundle, then exposes its value only as the provider's recognized child variable (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or `ZAI_API_KEY`). Neither the generic file path nor deprecated `PI_PROVIDER_API_KEY` is inherited by Pi. Local providers such as Ollama require no model key. -`THT_MODEL_API_KEY_FILE` supports Pi providers whose authentication is exactly one key: +`THT_MODEL_API_KEY` supports Pi providers whose authentication is exactly one key: `ant-ling`, `anthropic`, `cerebras`, `deepseek`, `fireworks`, `github-copilot`, `google` (including the `gemini` alias), `google-vertex` when using its API-key mode, `groq`, `huggingface`, `kimi-coding`, `minimax`, `minimax-cn`, `mistral`, `moonshotai`, diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 4f5cb0e9..58898e8b 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -1,55 +1,45 @@ -# Runtime secrets and private CA +# Runtime secrets -Secret values in this directory are ignored by Git and remain local to the cloned `ThothII` -directory. This self-contained layout is the default installation documented in -`docs/installazione-docker-4-contesti.md`; an enterprise deployment may point the same -`*_SECRET_FILE` variables at an external secret-manager materialization instead. - -Compose mounts each file read-only beneath `/run/secrets`. The core process runs as UID 10001; -the mounted files must be readable by that UID. Docker Compose file-backed secrets are normally -mounted read-only with mode `0444`. This mode is accepted only for runtime paths beneath -`/run/secrets`, where the container mount is read-only and scoped to services that declare the -secret. Source files on the host must have no group/other bits (`0600` or `0400`). Verify with: +The canonical deployment secret is the single local file +`deploy/secrets/thothii.secrets`. Copy the tracked template and protect the copy: ```sh -docker compose -f compose.yaml -f deploy/compose.production.yaml \ - --profile external run --rm core sh -c 'id && test -r /run/secrets/thoth_ca.pem' +cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets +chmod 600 deploy/secrets/thothii.secrets ``` -The CA file should contain only the public PEM certificate chain. API-key files should contain -one value with no surrounding quotes. +The file uses strict `KEY=VALUE` lines (comments and blank lines are allowed). The supported +keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_VEC_API_KEY`, +`THT_VEC_WRITE_API_KEY`, and the four `THT_VECTOR_*_PASSWORD` role passwords. Values must be +non-empty and contain no whitespace. Do not put secrets in the root `.env`, workspace YAML, +URLs, logs, or `docker compose config` output. -`THT_MODEL_API_KEY_SECRET_FILE` supplies one generic hosted-model key to the core. The backend -reads it afresh for each Pi child and maps it to the selected provider's native environment name; -the generic path/value is not placed in settings, health output, argv, or logs. Supported hosted -providers include Anthropic, OpenAI, Google/Gemini, DeepSeek, Z.AI, Groq, Mistral, OpenRouter, -xAI, and Cerebras. Local Ollama/LM Studio providers require no file. Compound providers such as -Bedrock, Azure OpenAI Responses, and Cloudflare Workers AI/Gateway fail closed because they -require multiple credential/configuration values. Unknown hosted providers fail closed until an -explicit mapping is added. - -## Rotating the initialized local-vector bootstrap password - -Replacing `THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE` or changing its contents does **not** rotate -an initialized PostgreSQL cluster. Use the supported workflow against the running local-vector -project: +Compose mounts the bundle read-only as `/run/secrets/thothii.secrets`. The host file must be a +regular non-symlink file with mode `0600` or `0400`; Docker's normal `0444` mode is accepted +only for the runtime mount beneath `/run/secrets`. The core runs as UID 10001. Verify the mount +without printing its contents: ```sh -./scripts/vector-rotate-bootstrap-password.sh \ - deploy/secrets/vector_bootstrap_password \ - deploy/secrets/vector_bootstrap_password.next +docker compose run --rm core sh -c 'id && test -r /run/secrets/thothii.secrets' ``` -The command authenticates using the current file, changes only the authenticated bootstrap role, -verifies a new login, and only then atomically replaces the current deployment secret file. If old -authentication or new-login verification fails, it exits without changing the deployment file; -verification failure also attempts to restore the old database password over the still-open -authenticated connection. After success, run the printed `vector-reconcile`/migration/core command. +A private CA PEM chain is not a bundle value: PEM whitespace is rejected by the strict parser. +Keep it in the host or secret manager and add a reviewed Compose override that mounts it at +`/run/secrets/ca-chain.pem` and sets `THT_SSL_CA` (or the adapter-specific setting). The base +Compose files intentionally do not create this mount. -`THT_VECTOR_BOOTSTRAP_USER` is authoritative for database initialization, reconciliation, and -rotation; non-default bootstrap role names are supported. Bootstrap, migrator, reader, and writer -secret files must be non-empty and contain no whitespace (including trailing newlines). Rotation -rejects invalid files before contacting PostgreSQL or staging a deployment-file replacement. +## Migration from separate secret files -Keep the staged new file on the same trusted host, mode `0600`, and retain a secure backup until the -post-rotation reconciliation and application health checks pass. +Older installations used `THT_*_SECRET_FILE` variables and one file per value. Migrate by +copying each value to its bundle key, validating with `docker compose config --quiet`, and only +then deleting the old files. The old variables remain a compatibility path for staged upgrades, +but the documented and tested default is `THT_SECRETS_FILE=deploy/secrets/thothii.secrets`. + +The local-vector bootstrap rotation helper still accepts an old/new password file as its +maintenance interface. Run it only with files protected by `0600`, then copy the resulting +password into `THT_VECTOR_BOOTSTRAP_PASSWORD` in the bundle before restarting +`vector-reconcile`/the application. The helper never prints password contents. + +Hosted Pi providers must use a single provider key. Compound providers (Bedrock, Azure OpenAI +Responses, Cloudflare Workers AI/Gateway) fail closed until a provider-specific credential +adapter is implemented. diff --git a/docs/index.md b/docs/index.md index 0858f9b1..fa2a74c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,9 @@ La documentazione è divisa in due aree: Come funziona il sistema: architettura, specifiche di design delle singole funzionalità, piani di implementazione, report di test. Parte da qui: [Panoramica dell'architettura](architecture/overview.md). -Per installare l'applicazione in Docker nei diversi contesti operativi: [Installazione Docker nei quattro contesti](installazione-docker-4-contesti.md). +Per installare l'applicazione in Docker nei quattro contesti operativi, partendo dal comando +predefinito `docker compose up --build -d` e dal bundle unico dei secret: +[Installazione Docker nei quattro contesti](installazione-docker-4-contesti.md). ## Considerazioni Generali diff --git a/docs/installazione-docker-4-contesti.md b/docs/installazione-docker-4-contesti.md index 6e8570b0..ea38f1dd 100644 --- a/docs/installazione-docker-4-contesti.md +++ b/docs/installazione-docker-4-contesti.md @@ -1,433 +1,227 @@ # Installazione Docker nei quattro contesti operativi -Questa guida descrive l'installazione di ThothII usando le due immagini applicative: +ThothII viene distribuito con due immagini applicative: - `thothii-core`: backend Fastify, harness `tht` e Pi; - `thothii-frontend`: frontend React servito da nginx. -I database e i servizi di embedding restano esterni, salvo il profilo opzionale -`local-vector`, che avvia un PostgreSQL/pgvector nello stesso progetto Compose. +PostgreSQL/pgvector, DWH ed Evidence restano esterni nel profilo predefinito. Il profilo opzionale `local-vector` avvia PostgreSQL/pgvector nel progetto Compose. -## Prerequisiti comuni +## Installazione comune (il comando standard) -Installare Docker Engine/Compose v2 sul server oppure Docker Desktop su macOS/Windows. -La macchina deve poter raggiungere: - -- il DWH/DWH REST, se usato dal workspace; -- il vector DB REST o pgvector; -- il servizio di embedding (Ollama o endpoint compatibile); -- il provider del modello Pi, se si usano provider hosted. - -Clonare il repository e lavorare dalla sua radice: +Servono Docker Engine/Compose v2 su Linux oppure Docker Desktop su macOS/Windows. Dalla directory in cui si vuole conservare il clone: ```sh git clone ThothII cd ThothII -cp deploy/env.example deploy/.env +cp .env.example .env mkdir -p deploy/secrets deploy/workspaces -chmod 600 deploy/.env +cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets +chmod 600 deploy/secrets/thothii.secrets ``` -La configurazione installativa rimane tutta dentro `ThothII/`: +Modificare **solo** questi file interni al clone: -```text -ThothII/ -├── deploy/.env # riferimenti ai file e valori non riservati -├── deploy/secrets/ # file secret locali, esclusi da Git -└── deploy/workspaces/ # workspace YAML senza password/token -``` +| File | Cosa contiene | +|---|---| +| `.env` | endpoint, database, provider, `COMPOSE_FILE` e `COMPOSE_PROFILES`; mai password/token | +| `deploy/secrets/thothii.secrets` | un bundle `NOME=VALORE`, mode host `0600` o `0400` | +| `deploy/workspaces/.yaml` | adapter, endpoint non riservati, `roots` ed Evidence | -I soli file da creare o modificare dopo il clone sono questi: - -| File interno a `ThothII/` | Operazione | Contenuto | -|---|---|---| -| `deploy/.env` | copiare da `deploy/env.example` e modificare | endpoint, nomi, profili e percorsi relativi dei secret | -| `deploy/secrets/dwh-api-key` | creare | una API key DWH, una riga/valore | -| `deploy/secrets/dwh_password` | creare solo con `postgres_direct` | password dell'utente DWH diretto | -| `deploy/secrets/vector-reader-api-key` | creare | chiave reader vector REST, se usata | -| `deploy/secrets/vector-writer-api-key` | creare | chiave writer vector REST, se usata | -| `deploy/secrets/model-api-key` | creare | chiave del provider Pi a chiave singola | -| `deploy/secrets/ca-chain.pem` | creare, se serve TLS privato | catena CA PEM pubblica | -| `deploy/secrets/vector_bootstrap_password` | creare con `local-vector` | password bootstrap pgvector | -| `deploy/secrets/vector_migrator_password` | creare con `local-vector` | password migrator | -| `deploy/secrets/vector_reader_password` | creare con `local-vector` | password reader pgvector | -| `deploy/secrets/vector_writer_password` | creare con `local-vector` | password writer pgvector | -| `deploy/workspaces/.yaml` | copiare/modificare | adapter, endpoint non riservati, `roots` e sorgente Evidence | - -Non occorre creare file in `/etc`, `/opt`, `~/.thothii` o in altre directory esterne per seguire -questa guida. I file secret sono ignorati da Git tramite `.gitignore`, ma restano disponibili a -Docker perché Compose li legge dal progetto locale. - -I secret non devono essere inseriti in `deploy/.env`, nei file workspace o negli URL. -`deploy/.env` contiene solo host, porte, nomi di database e percorsi relativi come -`THT_DWH_API_KEY_SECRET_FILE=deploy/secrets/dwh-api-key`. Un workspace YAML deve contenere al -massimo un riferimento come `password_file`, mai la password stessa; gli URL non devono -contenere user, password o token. Dopo aver modificato `deploy/.env`, caricarlo nella shell da -cui si eseguono i comandi Docker: +Il file `.env` viene caricato automaticamente da Docker Compose perché è nella radice del progetto. Il valore predefinito è `COMPOSE_FILE=compose.yaml`, con profili vuoti e `THT_SECRETS_FILE=deploy/secrets/thothii.secrets`. Perciò, dopo aver compilato `.env`, il bundle e almeno il workspace, l'avvio normale è sempre: ```sh -set -a -. ./deploy/.env -set +a +docker compose up --build -d ``` -Le tre righe fanno questo: +Non occorre usare `--env-file`, `-f` o `--profile` per questa installazione. Verificare lo stato con `docker compose ps` e aprire . `docker compose down` conserva il volume `thoth_data`; usare `down --volumes` solo per un ambiente effimero. -1. `set -a` dice alla shell di esportare automaticamente tutte le variabili assegnate da - questo momento in poi; -2. `. ./deploy/.env` (il punto iniziale è l'abbreviazione di `source`) legge ed esegue il file - nella shell corrente. Le variabili diventano quindi disponibili a `docker compose` e ai - processi figli, senza aprire una nuova shell; -3. `set +a` disattiva l'esportazione automatica per evitare che assegnazioni successive vengano - esportate accidentalmente. +### Formato del bundle unico -Esempio: se `deploy/.env` contiene +`deploy/secrets/thothii.secrets` è un file di testo locale, non uno script shell. Sono ammessi commenti e righe vuote; ogni altra riga deve essere una sola assegnazione senza spazi: ```dotenv -THT_MODEL_API_KEY_SECRET_FILE=deploy/secrets/model-api-key +THT_MODEL_API_KEY=... +THT_DWH_API_KEY=... +THT_VEC_API_KEY=... +THT_VEC_WRITE_API_KEY=... +THT_VECTOR_BOOTSTRAP_PASSWORD=... +THT_VECTOR_MIGRATOR_PASSWORD=... +THT_VECTOR_READER_PASSWORD=... +THT_VECTOR_WRITER_PASSWORD=... ``` -dopo il comando `source` la shell possiede `THT_MODEL_API_KEY_SECRET_FILE` e Compose può usare -quel percorso per leggere il secret file. Il valore della API key non viene caricato in -`deploy/.env`: resta nel file separato sotto `deploy/secrets/`. +Inserire solo le chiavi necessarie al profilo scelto. Il bundle viene montato in sola lettura nel container come `/run/secrets/thothii.secrets`; il parser rifiuta duplicati, chiavi sconosciute, valori vuoti, symlink e permessi host troppo aperti. Non inserire secret in `.env`, nei workspace, negli URL o nell'output di `docker compose config`. -La forma equivalente, senza modificare l'ambiente della shell, è passare il file direttamente a -Compose: +Una catena CA PEM **non può essere inserita nel bundle**: contiene whitespace e viene rifiutata dal parser. Se un endpoint usa una CA privata, conservarla nel secret manager/host e aggiungere un override Compose revisionato che monti il file in `/run/secrets/ca-chain.pem` e imposti `THT_SSL_CA` (o il parametro dell'adapter). Il clone base non crea quel mount: questa è una limitazione intenzionale da considerare in fase di deployment. -```sh -docker compose --env-file deploy/.env -f compose.yaml -f deploy/compose.production.yaml \ - --profile external up --build --wait -``` +### Overlay opzionali tramite `.env` -Non usare `source`/`.` con file ricevuti da terzi senza averne verificato il contenuto: un file -`.env` sourced è codice shell, non un semplice formato dati. In questa guida il file viene creato -localmente da `deploy/env.example` e contiene solo assegnazioni di configurazione e percorsi. - -Per i comandi successivi si può quindi scegliere una sola delle due modalità: fare `source` -una volta all'inizio della sessione, oppure aggiungere `--env-file deploy/.env` a ogni comando -`docker compose`. - -Salvare invece ogni credenziale in un file separato dentro `ThothII/deploy/secrets/`. Sul sistema -host il file deve essere leggibile solo dall'utente/servizio che esegue Docker (`0400` se solo -lettura, `0600` se l'operatore deve poterlo aggiornare). Compose lo monta poi nel container come secret -read-only sotto `/run/secrets`. Il `0444` osservabile dentro il container è normale per il -mount dei secret Docker e non rende il file host pubblico: il file host resta protetto e il -container riceve una copia/mount temporaneo non scrivibile. - -Il modello Pi usa `THT_MODEL_API_KEY_SECRET_FILE` per i provider autenticati con una singola -chiave. Provider composti (Bedrock, Azure OpenAI Responses, Cloudflare Workers AI/Gateway) -sono rifiutati esplicitamente: richiedono un bundle di credenziali non rappresentabile da un -solo file. - -## 1. Server applicativo insieme a DWH e vector DB - -Questo profilo è adatto quando ThothII, il database relazionale e il vector DB sono nella -stessa rete/server, ma i database non devono necessariamente essere containerizzati da -ThothII. Si usa il profilo `external`; gli endpoint possono essere nomi DNS, IP privati o -nomi di servizio della rete Docker. - -### Configurazione - -Prima si definiscono in `ThothII/deploy/.env` i percorsi host nelle variabili -`THT_*_SECRET_FILE`: +Gli overlay non cambiano il comando operativo. Impostare in `.env`: ```dotenv -THT_DWH_API_KEY_SECRET_FILE=deploy/secrets/dwh-api-key -THT_VEC_API_KEY_SECRET_FILE=deploy/secrets/vector-reader-api-key -THT_VEC_WRITE_API_KEY_SECRET_FILE=deploy/secrets/vector-writer-api-key -THT_MODEL_API_KEY_SECRET_FILE=deploy/secrets/model-api-key -THT_CA_SECRET_FILE=deploy/secrets/ca-chain.pem +# DWH/vector/embedding remoti (server applicativo o server con i DB): +COMPOSE_FILE=compose.yaml:deploy/compose.production.yaml +COMPOSE_PROFILES= + +# pgvector locale (Mac, Windows o server autonomo): +COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml +COMPOSE_PROFILES=local-vector ``` -Queste righe vanno scritte nel file interno `ThothII/deploy/.env`, non eseguite solo -temporaneamente nella shell. +Su Windows usare `;` come separatore di `COMPOSE_FILE`. Per il preprocessing locale aggiungere `deploy/compose.preprocess.yaml:deploy/compose.preprocess-local-vector.yaml` e impostare `COMPOSE_PROFILES=local-vector,preprocess`; poi usare `docker compose run --rm preprocess-evidence` oppure `docker compose run --rm preprocess-dwh`. -Nella forma documentale, i cinque target sono indicati come -``, ``, -``, `` e -``. Le parentesi angolari sono segnaposto e **non** vanno digitate nella -shell: nei comandi eseguibili si usano le variabili tra virgolette doppie: +## Workspace, adapter e Evidence -```sh -for secret_file in \ - "$THT_DWH_API_KEY_SECRET_FILE" "$THT_VEC_API_KEY_SECRET_FILE" \ - "$THT_VEC_WRITE_API_KEY_SECRET_FILE" "$THT_MODEL_API_KEY_SECRET_FILE" "$THT_CA_SECRET_FILE"; do - sudo install -d -m 700 "$(dirname "$secret_file")" -done -sudo install -m 600 /dev/null "$THT_DWH_API_KEY_SECRET_FILE" -sudo install -m 600 /dev/null "$THT_VEC_API_KEY_SECRET_FILE" -sudo install -m 600 /dev/null "$THT_VEC_WRITE_API_KEY_SECRET_FILE" -sudo install -m 600 /dev/null "$THT_MODEL_API_KEY_SECRET_FILE" -sudo install -m 600 /dev/null "$THT_CA_SECRET_FILE" -``` - -Così ogni `sudo install` usa la variabile corretta e non si rischia di creare il file in un -percorso diverso da quello che Compose leggerà. Se i secret sono forniti da un secret manager, -si impostano le stesse variabili ai file materializzati dal manager e si saltano i comandi -`install`. - -`install` è un comando Unix per creare/copiare un file impostando contestualmente i permessi. -In `install -m 600 /dev/null DEST`: - -- `/dev/null` è una sorgente vuota, quindi il comando crea (o sostituisce) `DEST` senza - inserire una password; -- `-m 600` imposta i permessi `rw-------` (lettura/scrittura solo per il proprietario); -- `DEST` è il file che l'operatore deve poi riempire con il secret. - -Dopo la creazione, scrivere il contenuto in ogni target (l'esempio seguente usa un file sorgente -protetto; ripeterlo per le altre variabili): - -```sh -sudo sh -c 'umask 077; cat > "$1"' sh "$THT_DWH_API_KEY_SECRET_FILE" \ - < /percorso/protetto/dwh-api-key -``` - -Il percorso `deploy/secrets/` è la directory interna al clone usata da questa guida: il software -non cerca automaticamente le API key in `/etc` né in una directory speciale. Il percorso host è -quello indicato nelle variabili `THT_*_SECRET_FILE`; Compose legge quel file e lo monta nel -container al percorso interno dichiarato dal servizio, normalmente `/run/secrets/dwh_api_key`, -`/run/secrets/vector_reader_api_key` o `/run/secrets/model_api_key`. - -Il comando non è un gestore di password e non va usato per stampare il secret sulla riga di -comando. Su Windows è preferibile usare WSL2 per creare il file con permessi Unix, oppure -creare un file locale ACL-protetto tramite uno strumento aziendale di gestione dei secret. Non -usare `ConvertFrom-SecureString` come contenuto del file: produce una rappresentazione cifrata -che ThothII non può usare come API key. - -```sh -# Dentro WSL2, dalla directory radice del clone ThothII. -mkdir -p deploy/secrets -umask 077 -read -r -s secret; printf '%s' "$secret" > deploy/secrets/dwh-api-key -unset secret -``` - -Per i secret usati da Docker Desktop è preferibile una directory locale non sincronizzata e -accessibile a Docker Desktop; non usare una cartella Git o OneDrive condivisa. In alternativa, -creare i file dentro WSL2 con `umask 077` e passare a Compose il path Windows risultante. - -Impostare gli endpoint e i riferimenti ai secret nel servizio core (preferibilmente tramite un file `.env` -gestito dall'operatore, non committato): - -```sh -export THT_DB_NAME=warehouse -export THT_DWH_REST_URL=https://dwh.internal.example -export THT_VEC_REST_URL=https://vectors.internal.example -export THT_OLLAMA_URL=https://embeddings.internal.example -export AUTH_MODE=none -export THOTH_PUBLIC_EXPOSURE=false -``` - -Queste variabili con suffisso `_SECRET_FILE` sono input di Compose sul server host. Non vanno -confuse con le variabili `_FILE` viste dal processo dentro il container: ad esempio -`THT_MODEL_API_KEY_SECRET_FILE=deploy/secrets/model-api-key` viene trasformata dal file -`deploy/compose.production.yaml` in `THT_MODEL_API_KEY_FILE=/run/secrets/model_api_key`. -Il backend legge quindi `/run/secrets/model_api_key`, non il file host sotto `deploy/secrets`. - -Preparare un file YAML in `deploy/workspaces/`. Il workspace è la configurazione logica di una -installazione: seleziona gli adapter, gli endpoint non riservati e le radici persistenti. Per -esempio, con DWH REST e vector DB HTTP: +Il workspace YAML seleziona il trasporto disponibile. Esempio DWH REST e vector DB HTTP: ```yaml language: en dwh: type: thoth_rest database: {database: warehouse, schema: datawarehouse} - endpoint: {base_url: https://dwh.internal.example} + endpoint: {base_url: https://dwh.example.test} vectors: type: thoth_vector_http - reader: {base_url: https://vectors.internal.example} - writer: {base_url: https://vectors.internal.example} + reader: {base_url: https://vectors.example.test} + writer: {base_url: https://vectors.example.test} roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} evidence: {source_root: /data/source, evidence_dir: evidence} -embeddings: {base_url: https://embeddings.internal.example, model: nomodel, dim: 768} +embeddings: {base_url: https://embeddings.example.test, model: nomodel, dim: 768} ``` -Con DWH PostgreSQL e pgvector raggiungibili direttamente dalla rete Docker: +Esempio con accesso diretto a PostgreSQL e pgvector: ```yaml language: en dwh: type: postgres_direct - connection: {host: dwh.internal.example, database: warehouse, schema: public, + connection: {host: dwh.internal, database: warehouse, schema: public, user: thoth_reader, password_file: /run/secrets/dwh_password} vectors: type: pgvector_direct - reader: {host: vector.internal.example, database: thoth, schema: vectors, + reader: {host: vector.internal, database: thoth, schema: vectors, user: thoth_vector_reader, password_file: /run/secrets/vector_reader_password} - writer: {host: vector.internal.example, database: thoth, schema: vectors, + writer: {host: vector.internal, database: thoth, schema: vectors, user: thoth_vector_writer, password_file: /run/secrets/vector_writer_password} roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} ``` -I nomi dei `type` sono contratti applicativi, non descrizioni libere: usare quelli esposti da -`tht doctor` e dagli esempi del repository (`thoth_rest`, `thoth_vector_http`, -`postgres_direct`, `pgvector_direct`). `dwh.type` sceglie come interrogare il DWH; `vectors.type` -sceglie come leggere/scrivere il vector DB. Cambiare questi valori può richiedere anche campi -specifici dell'adapter e secret file coerenti. +Questo esempio mostra il contratto dell'adapter: i file indicati da `password_file` devono +essere montati da un override Compose approvato. Il profilo base monta soltanto il bundle unico; +per un DWH diretto occorre quindi materializzare il file password dal secret manager e aggiungere +il bind mount/runtime adapter corrispondente. Non inserire la password nel workspace o nell'URL. -`roots` contiene percorsi logici, non percorsi host arbitrari. Con `THT_DATA_ROOT=/data`, -`artifacts: artifacts` diventa `/data/workspaces//artifacts` (e analogamente per -`indexes` e `sessions`), evitando che un YAML possa scrivere fuori dal volume applicativo. -Un path host va esposto esplicitamente con un bind mount read-only/read-write nel Compose e poi -referenziato dal workspace secondo le regole di sicurezza; non inserire `/Users/...` o -`C:\\...` direttamente in un workspace destinato a più sistemi operativi. +`roots` sono relativi e vengono risolti sotto `/data/workspaces/` nel volume Docker; non inserire path host come `/Users/...` o `C:\\...`. Per Evidence usare una radice filesystem montata in sola lettura oppure l'adapter HTTP/S3 previsto dal workspace. Per HTTP/S3 definire allowlist, limiti di dimensione/paginazione e una politica egress; non mettere token nelle URI. -### Avvio e verifica +## 1. Server remoto insieme ai database e al vector DB -```sh -docker compose -f compose.yaml -f deploy/compose.production.yaml \ - --profile external up --build --wait +Usare quando il server Docker è nella stessa rete del DWH e del vector DB (containerizzati o meno). Il file `.env` può restare sul default, senza profili, impostando gli endpoint raggiungibili localmente: -docker compose -f compose.yaml -f deploy/compose.production.yaml \ - --profile external exec core /opt/venv/bin/tht doctor --json -./scripts/docker-smoke.sh +```dotenv +COMPOSE_FILE=compose.yaml +COMPOSE_PROFILES= +THT_DB_NAME=warehouse +THT_DWH_REST_URL=https://dwh.internal.example +THT_VEC_REST_URL=https://vectors.internal.example +THT_OLLAMA_URL=https://embeddings.internal.example +AUTH_MODE=none +THOTH_PUBLIC_EXPOSURE=false ``` -Il frontend è pubblicato su `127.0.0.1:8080` per default. Per esposizione pubblica usare un -reverse proxy autenticato TLS; non impostare `THOTH_PUBLIC_EXPOSURE=true` senza `AUTH_MODE=upstream` -e senza il proxy che inietta `X-Authenticated-User`. - -## 2. Mac locale con DWH esterno e vector DB locale - -Questo è il profilo consigliato per un Mac Apple Silicon o Intel: Docker Desktop ospita -pgvector e ThothII, mentre DWH ed embeddings possono essere remoti. Ollama eseguito sul Mac è -raggiungibile dai container con `host.docker.internal`. - -Creare quattro password locali dentro `ThothII/deploy/secrets/` (la directory è esclusa da Git): +Riempire nel bundle le chiavi DWH/vector/model necessarie e avviare: ```sh -mkdir -p deploy/secrets -for name in bootstrap migrator reader writer; do - umask 077; openssl rand -base64 32 >"deploy/secrets/$name" -done -export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE=deploy/secrets/bootstrap -export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE=deploy/secrets/migrator -export THT_VECTOR_READER_PASSWORD_SECRET_FILE=deploy/secrets/reader -export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE=deploy/secrets/writer -export THT_OLLAMA_URL=http://host.docker.internal:11434 +docker compose up --build -d +docker compose exec core /opt/venv/bin/tht doctor --json ``` -Avviare il profilo: +Se si abilita l'overlay production, il proxy autenticato TLS deve essere l'unico listener pubblico +e deve iniettare `X-Authenticated-User`; non esporre direttamente la porta pubblicata da nginx. -```sh -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - --profile local-vector up --build --wait +Se il server deve essere raggiungibile da altri host, sostituire `COMPOSE_FILE` con +`compose.yaml:deploy/compose.production.yaml`, configurare il proxy autenticato e impostare +`AUTH_MODE=upstream`/`THOTH_PUBLIC_EXPOSURE=true` come descritto nella sezione di trust boundary. + +## 2. Mac locale + +Installare Docker Desktop e, se usato, Ollama sul Mac. Nel `.env` selezionare il profilo locale: + +```dotenv +COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml +COMPOSE_PROFILES=local-vector +THT_OLLAMA_URL=http://host.docker.internal:11434 ``` -Per eseguire i job di preprocessing usare anche l'overlay che abilita la catena -health-check → reconcile → migrate: +Nel bundle aggiungere quattro password generate localmente: -```sh -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess build preprocess-evidence -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess run --rm preprocess-evidence +```dotenv +THT_VECTOR_BOOTSTRAP_PASSWORD= +THT_VECTOR_MIGRATOR_PASSWORD= +THT_VECTOR_READER_PASSWORD= +THT_VECTOR_WRITER_PASSWORD= ``` -Montare i testi Evidence in `/data/source/evidence` tramite il workspace o un override Compose. -Il volume `thoth_data` contiene sessioni, artefatti e corpus; `vector_data` contiene solo -pgvector. Per verificare rotazione credenziali e recovery: - -```sh -./scripts/local-vector-smoke.sh -./scripts/local-vector-smoke.sh --backup-restore -``` +Poi eseguire il comando standard `docker compose up --build -d`. Il primo avvio esegue reconciliation dei ruoli e migrazione pgvector. Per preprocessing, impostare il preset indicato sopra e usare `docker compose run --rm preprocess-evidence`/`preprocess-dwh`. ## 3. PC Windows locale -Usare Docker Desktop con backend WSL2, abilitare l'integrazione con la distribuzione WSL e -conservare il repository in un percorso condiviso con Docker. È preferibile lavorare da -PowerShell nella directory del progetto. +Usare Docker Desktop con backend WSL2 e abilitare la condivisione della directory del clone. Modificare `.env` con il separatore Windows: -```powershell -$secretDir = Join-Path (Get-Location) "deploy\secrets" -New-Item -ItemType Directory -Force $secretDir | Out-Null -foreach ($name in @("bootstrap", "migrator", "reader", "writer")) { - $bytes = New-Object byte[] 32 - [Security.Cryptography.RandomNumberGenerator]::Fill($bytes) - [Convert]::ToBase64String($bytes) | Set-Content -NoNewline (Join-Path $secretDir $name) -} -$env:THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE = Join-Path $secretDir "bootstrap" -$env:THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE = Join-Path $secretDir "migrator" -$env:THT_VECTOR_READER_PASSWORD_SECRET_FILE = Join-Path $secretDir "reader" -$env:THT_VECTOR_WRITER_PASSWORD_SECRET_FILE = Join-Path $secretDir "writer" -$env:THT_OLLAMA_URL = "http://host.docker.internal:11434" +```dotenv +COMPOSE_FILE=compose.yaml;deploy/compose.local-vector.yaml +COMPOSE_PROFILES=local-vector +THT_OLLAMA_URL=http://host.docker.internal:11434 ``` -Da PowerShell, i path assoluti vengono passati a Compose tramite le variabili precedenti; non -scrivere password direttamente nello YAML. Avviare lo stesso profilo del Mac: +Creare `deploy/secrets/thothii.secrets` con un editor locale protetto (ACL leggibile solo dall'utente Docker) e le stesse quattro chiavi pgvector del profilo Mac. Non usare `ConvertFrom-SecureString`: il bundle deve contenere il valore in chiaro per il servizio, con accesso limitato al file. Da PowerShell, dalla radice del clone, eseguire: ```powershell -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml ` - --profile local-vector up --build --wait -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml ` - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml ` - --profile local-vector --profile preprocess run --rm preprocess-evidence +docker compose up --build -d +docker compose ps ``` -Se Docker Desktop segnala un bind mount non condiviso, aggiungere la directory del repository -in Docker Desktop → Settings → Resources → File Sharing. Se Ollama gira su Windows, usare -`host.docker.internal`; se gira in WSL2, usare l'endpoint raggiungibile dalla rete Docker. -Verificare il profilo con: +Se un bind mount viene rifiutato, aggiungere la cartella del repository a Docker Desktop → Settings → Resources → File Sharing. Per Ollama eseguito in WSL2 usare l'indirizzo raggiungibile dalla rete Docker invece di assumere `localhost`. -```powershell -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml --profile local-vector config -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml --profile local-vector ps +## 4. Server applicativo distinto da DB ed Evidence + +Usare il profilo production e consentire dal firewall solo le destinazioni necessarie: + +```dotenv +COMPOSE_FILE=compose.yaml:deploy/compose.production.yaml +COMPOSE_PROFILES= +THT_DWH_REST_URL=https://dwh.example.test +THT_VEC_REST_URL=https://vectors.example.test +THT_OLLAMA_URL=https://embeddings.example.test ``` -## 4. Server applicativo distinto da DB e Evidence +Il DWH e il vector DB possono essere REST/HTTP oppure adapter diretti (`postgres_direct`, `pgvector_direct`) se il server ha connettività TCP. Le Evidence possono essere: -Questo profilo separa il server Docker di ThothII dal DWH, vector DB e repository Evidence. -Usare `compose.production.yaml` e configurare gli adapter con endpoint raggiungibili dal -server applicativo. Le Evidence non devono essere copiate nel container se sono già accessibili -via HTTP/S3 o tramite un filesystem montato dal sistema operativo. +- filesystem NFS/SMB montato sul server e presentato come root read-only; +- endpoint HTTPS, con allowlist e limiti SSRF; +- bucket S3 con secret references e endpoint custom esplicitamente autorizzati. -Configurare firewall e DNS in modo che il server applicativo possa raggiungere solo gli endpoint -necessari. Esempi di sorgente Evidence: +Il preprocessing può girare sul server applicativo usando il volume `/data`; mantenere separati workspace, lock e artefatti dei job. Avviare con il comando standard e verificare `tht doctor`. -- `filesystem`: NFS/SMB montato sul server e passato come root read-only al workspace; -- `http`: URL HTTPS con allowlist/limiti SSRF e cache condizionata; -- `s3`: bucket/prefix con secret references e endpoint custom solo con trust boundary esplicito. +## Migrazione da installazioni con secret separati -Impostare `THT_DOCS_ROOT`/il workspace per il path montato oppure configurare il tipo HTTP/S3, -poi avviare: +Le variabili `THT_*_SECRET_FILE` e i file `dwh-api-key`, `vector-reader-api-key`, `vector-writer-api-key`, `model-api-key` e `vector_*_password` appartengono al layout precedente. Non vengono importati automaticamente dal bundle. Per migrare: + +1. creare `deploy/secrets/thothii.secrets` mode `0600`; +2. copiare ogni valore nel nome chiave corrispondente (`THT_DWH_API_KEY`, `THT_VEC_API_KEY`, `THT_VEC_WRITE_API_KEY`, `THT_MODEL_API_KEY` o `THT_VECTOR_*_PASSWORD`), senza virgolette né newline; +3. rimuovere dal `.env` le variabili `_SECRET_FILE` e impostare `THT_SECRETS_FILE` al percorso del bundle (il default relativo è già corretto); +4. eseguire `docker compose config --quiet` e poi `docker compose up --build -d`; +5. solo dopo la verifica, cancellare i vecchi file separati. + +Una CA PEM resta un'eccezione esterna come descritto sopra. Provider Pi con credenziali composte (Bedrock, Azure OpenAI Responses, Cloudflare Workers AI/Gateway) restano rifiutati finché non viene implementato un adapter dedicato. + +## Controlli post-installazione ```sh -docker compose -f compose.yaml -f deploy/compose.production.yaml \ - --profile external config --quiet -docker compose -f compose.yaml -f deploy/compose.production.yaml \ - --profile external up --build --wait -docker compose -f compose.yaml -f deploy/compose.production.yaml \ - --profile external exec core /opt/venv/bin/tht doctor --json +docker compose config --quiet +docker compose ps +docker compose exec core /opt/venv/bin/tht doctor --json +./scripts/docker-smoke.sh ``` -Per un vector DB remoto usare le chiavi reader/writer distinte quando il servizio lo consente; -per un DWH REST usare `THT_DWH_REST_URL` e il relativo secret, senza introdurre connessioni -dirette nel container. Il preprocessing DWH/Evidence può essere eseguito sullo stesso server -applicativo con un volume `/data` persistente, mantenendo separati i lock e gli artefatti dei -job. - -## Controlli comuni post-installazione - -1. `docker compose ... config --quiet` non deve mostrare password o token in chiaro. -2. `docker compose ... ps` deve mostrare `core` e `frontend` healthy. -3. `tht doctor --json` deve restituire JSON valido; errori di dipendenza devono restare - diagnostici e non esporre secret. -4. Eseguire lo smoke coerente col profilo (`docker-smoke.sh`, `preprocess-smoke.sh` o - `local-vector-smoke.sh`). -5. Conservare il volume `thoth_data` e i backup pgvector fuori dal ciclo di deploy; usare - `down --volumes` solo per ambienti effimeri o dopo aver verificato il backup. - -## Risoluzione rapida dei problemi - -- **Core healthy ma nessun modello:** controllare `THT_MODEL_API_KEY_SECRET_FILE`, il provider - selezionato e che non sia uno dei provider composti non supportati. -- **Preprocess fallisce subito:** verificare l'overlay `compose.preprocess-local-vector.yaml`, - i quattro secret pgvector e che il mount Evidence sia leggibile. -- **Vector health non disponibile:** controllare `vector-db`, `vector-reconcile`, `vector-migrate` - e le password reader/writer; non usare la password bootstrap nelle query applicative. -- **Evidence non trovate:** controllare il path nel workspace, la rete HTTP/S3 e i limiti di - dimensione/paginazione; verificare che il corpus ACTIVE appartenga allo stesso workspace. +Per il profilo locale usare anche `./scripts/local-vector-smoke.sh`; per il preprocessing `./scripts/preprocess-smoke.sh`. Non pubblicare `.env` o `deploy/secrets/thothii.secrets` nei log, nei backup Git o nei ticket. diff --git a/scripts/docker-smoke.sh b/scripts/docker-smoke.sh index 0c79468f..f2a1ae31 100755 --- a/scripts/docker-smoke.sh +++ b/scripts/docker-smoke.sh @@ -7,6 +7,11 @@ marker="smoke-$(date +%s)-$$" headers="" smoke_project=${SMOKE_PROJECT:-"thothii-smoke-$(date +%s)-$$"} keep_resources=${KEEP_SMOKE_RESOURCES:-0} +bundle=$(mktemp) +printf '%s\n' '# disposable smoke bundle' 'THT_MODEL_API_KEY=smoke-model-key' >"$bundle" +chmod 0600 "$bundle" +export THT_SECRETS_FILE="$bundle" +trap 'rm -f "$bundle"' EXIT HUP INT TERM case "$smoke_project" in thothii) @@ -20,7 +25,7 @@ case "$smoke_project" in esac compose() { - docker compose --project-name "$smoke_project" --profile external "$@" + docker compose --project-name "$smoke_project" "$@" } # Avoid colliding with a developer's existing service. Production/developer Compose still # defaults to 8080; a published port of 0 asks Docker for a free ephemeral smoke port. @@ -28,6 +33,7 @@ export THOTH_HTTP_PORT=${THOTH_HTTP_PORT:-0} cleanup() { if [ -n "$headers" ]; then rm -f "$headers"; fi + rm -f "$bundle" if [ "$keep_resources" = "1" ]; then echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2 else diff --git a/scripts/test-default-compose.sh b/scripts/test-default-compose.sh index 97540ba3..b9fcd803 100755 --- a/scripts/test-default-compose.sh +++ b/scripts/test-default-compose.sh @@ -3,13 +3,22 @@ set -eu cd "$(dirname "$0")/.." +test -f .env.example +test -f deploy/secrets/thothii.secrets.example +grep -q '^docker compose up --build -d$' docs/installazione-docker-4-contesti.md +if grep -q 'cp deploy/env.example deploy/.env\|THT_[A-Z0-9_]*_SECRET_FILE=' docs/installazione-docker-4-contesti.md; then + echo "installation guide still presents the legacy per-file secret setup" >&2 + exit 1 +fi + tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT HUP INT TERM mkdir -p "$tmp/deploy/secrets" "$tmp/deploy/workspaces" cp compose.yaml "$tmp/compose.yaml" cp .env.example "$tmp/.env" -printf '%s\n' 'THT_MODEL_API_KEY=example-secret' >"$tmp/deploy/secrets/thothii.secrets" +cp deploy/secrets/thothii.secrets.example "$tmp/deploy/secrets/thothii.secrets" +printf '%s\n' 'THT_MODEL_API_KEY=example-secret' >>"$tmp/deploy/secrets/thothii.secrets" chmod 0600 "$tmp/deploy/secrets/thothii.secrets" services=$(docker compose --project-directory "$tmp" config --services) diff --git a/scripts/test-docker-smoke.sh b/scripts/test-docker-smoke.sh index f7d01de9..0ae82c78 100755 --- a/scripts/test-docker-smoke.sh +++ b/scripts/test-docker-smoke.sh @@ -58,8 +58,8 @@ run_smoke thothii-smoke-dynamic while IFS= read -r invocation; do case "$invocation" in - "compose --project-name thothii-smoke-dynamic --profile external "*) ;; - *) echo "Compose invocation escaped the smoke project/profile: $invocation" >&2; exit 1 ;; + "compose --project-name thothii-smoke-dynamic "*) ;; + *) echo "Compose invocation escaped the smoke project: $invocation" >&2; exit 1 ;; esac done <"$log" grep -q ' down --volumes$' "$log"