From 351361f72fd9463d7a8f253b7073541ccf3caf6e Mon Sep 17 00:00:00 2001 From: mptyl Date: Sun, 16 Aug 2026 14:19:32 +0200 Subject: [PATCH] feat: finish Pi and workspace management updates --- deploy/pi/models.json | 4 +- deploy/pi/settings.json | 2 +- docs/contracts/thothctl-pi.md | 45 +- docs/install/pi-management.md | 62 +- docs/reports/2026-08-15-tht-command-audit.md | 171 +++ ...8-15-tht-command-maintain-erase-enhance.md | 148 +++ ...-08-14-thothctl-discovery-and-pi-update.md | 86 ++ ...2026-08-15-unified-tht-cli-product-step.md | 1159 +++++++++++++++++ ...thothctl-discovery-and-pi-update-design.md | 44 + ...-15-unified-tht-cli-product-step-design.md | 235 ++++ frontend/src/shell/PiManagement.test.tsx | 77 +- frontend/src/shell/PiManagement.tsx | 39 +- frontend/src/shell/WorkspaceManager.test.tsx | 61 +- frontend/src/shell/WorkspaceManager.tsx | 82 +- tools/tht/internal/output/sanitize.go | 6 + tools/tht/internal/output/sanitize_test.go | 20 + tools/tht/internal/pi/commands.go | 2 +- tools/tht/internal/pi/version.go | 46 + tools/tht/internal/pi/version_test.go | 52 + tools/tht/internal/workspaceops/operations.go | 84 +- 20 files changed, 2276 insertions(+), 149 deletions(-) create mode 100644 docs/reports/2026-08-15-tht-command-audit.md create mode 100644 docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md create mode 100644 docs/superpowers/plans/2026-08-14-thothctl-discovery-and-pi-update.md create mode 100644 docs/superpowers/plans/2026-08-15-unified-tht-cli-product-step.md create mode 100644 docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md create mode 100644 docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md create mode 100644 tools/tht/internal/pi/version.go create mode 100644 tools/tht/internal/pi/version_test.go diff --git a/deploy/pi/models.json b/deploy/pi/models.json index 1b49dac8..75e48255 100644 --- a/deploy/pi/models.json +++ b/deploy/pi/models.json @@ -6,8 +6,8 @@ "apiKey": "$ZAI_API_KEY", "models": [ { - "id": "glm-5.2", - "name": "GLM-5.2", + "id": "glm-5.3", + "name": "GLM-5.3", "reasoning": true, "contextWindow": 200000, "maxTokens": 131072 diff --git a/deploy/pi/settings.json b/deploy/pi/settings.json index 31a18ce0..08f6d1b4 100644 --- a/deploy/pi/settings.json +++ b/deploy/pi/settings.json @@ -1,7 +1,7 @@ { "defaultProjectTrust": "always", "enabledModels": [ - "zai/glm-5.2", + "zai/glm-5.3", "deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-pro", "aritmolab/qwen3.6-35b-a3b" diff --git a/docs/contracts/thothctl-pi.md b/docs/contracts/thothctl-pi.md index 9ff37c46..852a1ead 100644 --- a/docs/contracts/thothctl-pi.md +++ b/docs/contracts/thothctl-pi.md @@ -6,13 +6,18 @@ does not mount a Docker socket, and Pi is never updated in a running container. ## Inspection and configuration ```text -thothctl --installation /absolute/path/thothii-installation.yaml pi status -thothctl --installation /absolute/path/thothii-installation.yaml pi doctor -thothctl --installation /absolute/path/thothii-installation.yaml pi test -thothctl --installation /absolute/path/thothii-installation.yaml pi logs -thothctl --installation /absolute/path/thothii-installation.yaml pi configure +thothctl pi status +thothctl pi doctor +thothctl pi test +thothctl pi logs +thothctl pi configure ``` +When `--installation` is omitted, `thothctl` first uses `THOTHII_INSTALLATION` and otherwise +discovers one valid `thothii-installation.yaml` in the current project tree, including an immediate +`deploy/*` directory. Use `--installation /absolute/path/thothii-installation.yaml` as an explicit +override when the descriptor is outside that tree or more than one installation is available. + `status` executes the image-bundled `pi --version`. `doctor` compares that value with both the container's `PI_VERSION` contract and the `io.thothii.pi.version` image label; a merely nonempty version is not sufficient. `doctor` and `test` also require a healthy core, a successful Pi smoke, @@ -25,7 +30,7 @@ models come from the backend's closed model list, and the model choices are rest selected provider. In non-interactive use, all choices must be explicit: ```text -thothctl --installation /absolute/path/thothii-installation.yaml pi configure \ +thothctl pi configure \ --provider zai --model glm-5.2 --thinking medium ``` @@ -70,7 +75,7 @@ secret overrides must never bypass that preflight wrapper. Configuration reload is a separate lifecycle operation from an image update: ```text -thothctl --installation /absolute/path/thothii-installation.yaml pi restart --yes [--drain] +thothctl pi restart --yes [--drain] ``` `--yes` is required after reviewing the planned core recreation. Restart activates the durable @@ -104,13 +109,25 @@ recovery rather than deleting recovery material. ## Updating Pi -Every update requires a pinned version, an explicit source, and confirmation: +The normal update uses the repository's pinned version and build source automatically: ```text -thothctl --installation /absolute/path/thothii-installation.yaml pi update \ - --version 0.81.0 --source build --yes +thothctl pi update +thothctl pi update --version 0.81.0 +``` -thothctl --installation /absolute/path/thothii-installation.yaml pi update \ +With no `--version`, the command reads the single default `ARG PI_VERSION=` from +`docker/core.Dockerfile` in the selected project. The normal path confirms the explicit update +command, drains active sessions without terminating them, builds the candidate, recreates only +`core`, verifies it, and promotes it transactionally. + +Advanced registry updates remain available and require an immutable digest: + +```text +thothctl pi update \ + --version 0.81.0 --source build --yes --drain + +thothctl pi update \ --version 0.81.0 --source pull \ --image registry.example.invalid/thothii-core@sha256:<64-lowercase-hex-digits> --yes ``` @@ -178,7 +195,7 @@ gate can open. For a failed update with `update-state.json`, first run: ```text -thothctl --installation /absolute/path/thothii-installation.yaml pi rollback --yes +thothctl pi rollback --yes ``` Rollback restores the image recorded in update state, but it checks restart state before making any @@ -189,8 +206,8 @@ reported problem, then use maintenance recovery. Inspect and clean a stale durable gate with: ```text -thothctl --installation /absolute/path/thothii-installation.yaml pi maintenance status -thothctl --installation /absolute/path/thothii-installation.yaml pi maintenance recover --yes +thothctl pi maintenance status +thothctl pi maintenance recover --yes ``` `maintenance recover` restores the captured restart image pin and lifecycle override when needed, diff --git a/docs/install/pi-management.md b/docs/install/pi-management.md index e604bc36..a8cca479 100644 --- a/docs/install/pi-management.md +++ b/docs/install/pi-management.md @@ -4,14 +4,21 @@ ThothII bundles Pi in the `core` image. Operators use the Pi Management page for defaults and the host-side `thothctl` CLI for lifecycle work. A local Pi installation is not required. -In the commands below, replace `/absolute/path/to/thothii-installation.yaml` with the protected -installation descriptor created by the [local installation guide](local.md). +Run these commands from the root of the current ThothII checkout or worktree. `thothctl` discovers +the valid installation descriptor in that project tree, so it uses the `deploy/` files belonging to +the checkout from which you run it. Do not use `~/bin`: `~` is the user home directory, not the +project root. ```sh -THTCTL=/absolute/path/to/thothctl -INSTALLATION=/absolute/path/to/thothii-installation.yaml +mkdir -p bin +go -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl +THTCTL=./bin/thothctl ``` +If `thothctl` is already on `PATH`, you may use `THTCTL=thothctl` instead. For an installation +stored elsewhere, set `THOTHII_INSTALLATION` or pass +`--installation /thothii-installation.yaml` explicitly. + ## Choose application defaults Use the **Pi Management** page to select the supported provider, model, and reasoning default, then @@ -21,8 +28,8 @@ diagnostics; it never accepts or displays a credential, opens a terminal, or upd Alternatively, use the CLI from an administrator terminal: ```sh -"$THTCTL" --installation "$INSTALLATION" pi configure -"$THTCTL" --installation "$INSTALLATION" pi configure --provider zai --model glm-5.2 --thinking medium +"$THTCTL" pi configure +"$THTCTL" pi configure --provider zai --model glm-5.2 --thinking medium ``` Use GUI Save defaults or CLI `pi configure`, not both for the same change. The CLI's interactive @@ -32,11 +39,11 @@ methods store application defaults in backend installation settings, not in the Useful read-only checks are: ```sh -"$THTCTL" --installation "$INSTALLATION" pi status -"$THTCTL" --installation "$INSTALLATION" pi doctor -"$THTCTL" --installation "$INSTALLATION" pi test -"$THTCTL" --installation "$INSTALLATION" pi check -"$THTCTL" --installation "$INSTALLATION" pi logs +"$THTCTL" pi status +"$THTCTL" pi doctor +"$THTCTL" pi test +"$THTCTL" pi check +"$THTCTL" pi logs ``` `pi check` is an alias for `pi test`; logs are a sanitized, bounded snapshot with no follow mode. @@ -72,7 +79,7 @@ After changing the provider catalog, enabled-model policy, or selected credentia running application with one confirmed restart: ```sh -"$THTCTL" --installation "$INSTALLATION" pi restart --yes --drain +"$THTCTL" pi restart --yes --drain ``` `--yes` confirms that core will be recreated. Without `--drain`, restart refuses active sessions; @@ -89,18 +96,25 @@ and `thothctl start` or raw Compose commands for this reload workflow. ## Update the bundled Pi version -`pi update` is for a new bundled Pi version; it is not a configuration reload. Finish or drain -active work, then choose an explicit source and version. A build update uses this checkout: +`pi update` is for a new bundled Pi version; it is not a configuration reload. The simple command +uses the single `ARG PI_VERSION=...` pin in `docker/core.Dockerfile`, builds that version, waits for +active sessions to finish, and recreates only `core`: ```sh -"$THTCTL" --installation "$INSTALLATION" pi update \ - --version 0.81.0 --source build --yes --drain +"$THTCTL" pi update +``` + +To build a specific version, pass `--version`; source, confirmation, and drain are automatic for +this normal build path: + +```sh +"$THTCTL" pi update --version 0.81.0 ``` A registry update must use an immutable digest, never a mutable tag: ```sh -"$THTCTL" --installation "$INSTALLATION" pi update \ +"$THTCTL" pi update \ --version 0.81.0 --source pull \ --image registry.example.invalid/thothii-core@sha256:<64-lowercase-hex-digits> \ --yes --drain @@ -117,24 +131,24 @@ reported recovery state and transaction override. Do not delete `.thothctl`, sta containers, or volumes. Inspect status and sanitized logs: ```sh -"$THTCTL" --installation "$INSTALLATION" pi maintenance status -"$THTCTL" --installation "$INSTALLATION" pi status -"$THTCTL" --installation "$INSTALLATION" pi logs +"$THTCTL" pi maintenance status +"$THTCTL" pi status +"$THTCTL" pi logs ``` For a failed update, restore its prior image: ```sh -"$THTCTL" --installation "$INSTALLATION" pi rollback --yes +"$THTCTL" pi rollback --yes ``` For a failed restart, use maintenance recovery instead of rollback. After repairing the reported Docker, disk, or configuration problem, use the same command to complete either safe recovery path: ```sh -"$THTCTL" --installation "$INSTALLATION" pi maintenance recover --yes -"$THTCTL" --installation "$INSTALLATION" pi doctor -"$THTCTL" --installation "$INSTALLATION" pi test +"$THTCTL" pi maintenance recover --yes +"$THTCTL" pi doctor +"$THTCTL" pi test ``` `pi rollback --yes` restores the prior update image. `pi maintenance recover --yes` checks both diff --git a/docs/reports/2026-08-15-tht-command-audit.md b/docs/reports/2026-08-15-tht-command-audit.md new file mode 100644 index 00000000..d2cc3a00 --- /dev/null +++ b/docs/reports/2026-08-15-tht-command-audit.md @@ -0,0 +1,171 @@ +# Audit critico dei comandi `tht` + +Data: 2026-08-15 + +## Scopo + +Questo audit valuta tutti i comandi terminali registrati dall'attuale CLI Python `tht` prima di +unificare la CLI di ThothII sotto un solo eseguibile pubblico. La valutazione incrocia: + +- il contratto del workflow Pi in `harness/.pi/skills/tht-sessione/SKILL.md`; +- le invocazioni reali del gate in `harness/.pi/extensions/tht-gate.js`; +- le invocazioni del backend in `backend/src/tht/tht-runner.ts`; +- i job operatore in `backend/src/workspaces/preprocessing-service.ts`; +- il migratore in `docker/session-migrate.sh`; +- test e documentazione esistenti. + +L'inventario autorevole contiene 76 comandi Typer più il comando callback `doctor`: 77 comandi +terminali complessivi. + +## Legenda + +- **WF — intoccabile workflow**: chiamato da Pi, dal gate o dal contratto delle otto fasi. Va + conservato con semantica, output JSON ed exit code compatibili. Non deve necessariamente apparire + nell'help ordinario dell'utente. +- **PL — intoccabile piattaforma**: chiamato dal backend, dai job workspace o dal deployment. Anche + questo è un contratto interno, non necessariamente un comando da mostrare all'utente. +- **ADV — mantenere avanzato**: non è nel flusso automatico, ma offre una capacità amministrativa o + di recupero che sarebbe imprudente perdere. Va nascosto dall'help base. +- **ACCORPA**: la capacità serve, ma non merita un comando autonomo. +- **RIMUOVI**: il comando non ha chiamanti reali ed è duplicato, superato, pericoloso o incompleto. + L'eventuale logica riutilizzata da altri flussi resta una libreria interna. + +## Risultato sintetico + +| Esito | Numero | Conseguenza | +|---|---:|---| +| WF o PL, intoccabili | 55 | Conservare il contratto; nascondere i primitivi tecnici dall'help base | +| ADV o ACCORPA | 8 | Conservare la capacità riducendo la superficie UX | +| RIMUOVI | 14 | Eliminare il comando dalla nuova CLI | +| **Totale** | **77** | Una sola CLI pubblica molto più semplice, senza riscrivere il workflow vivo | + +## Matrice completa + +### Diagnostica, configurazione e dipendenze + +| Comando | Valutazione | +|---|---| +| `config check` | **ACCORPA** in `tht doctor`: la validazione della configurazione serve, ma due preflight distinti confondono l'utente. | +| `doctor` | **ACCORPA/MANTIENI pubblico** come unico `tht doctor`, includendo controlli host, Compose, storage e configurazione runtime. | +| `db ping` | **PL**: il backend lo usa per rifiutare correttamente una nuova sessione quando il DWH non è raggiungibile o non è read-only. Interno. | +| `db fetch-ca` | **ACCORPA** in `tht setup` o nella configurazione workspace: utile per TLS, ma non giustifica un comando isolato. | +| `ollama ensure` | **PL**: preflight automatico dell'embedder usato dal backend. Interno. | + +### Fasi e decision ledger + +| Comando | Valutazione | +|---|---| +| `phase advance` | **WF**: il gate lo usa per avanzare solo dopo la decisione umana. Primitivo anti-bypass, quindi interno. | +| `phase meta` | **WF**: fornisce al gate la definizione data-driven delle fasi e dei tipi di decisione. Interno. | +| `phase reopen` | **WF**: è il percorso canonico per tornare a una fase precedente e invalidare deterministicamente gli artefatti successivi. | +| `phase show` | **WF**: il gate lo usa per calcolare la fase corrente. Interno. | +| `decision add` | **WF**: persistenza fondamentale delle decisioni del reviewer. Solo gate, non shell utente. | +| `decision add-batch` | **WF**: scrittura atomica delle decisioni multiple. Evita ledger parziali. | +| `decision add-join-set` | **WF**: sostituzione atomica dell'intero insieme di join. | +| `decision list` | **RIMUOVI**: nessun chiamante; `session show --json` contiene già il ledger necessario. | +| `decision retract` | **RIMUOVI** dalla CLI: nessun flusso vivo lo invoca e `phase reopen` è il percorso di correzione supportato. La semantica tombstone può restare nel dominio finché utile. | + +### Sessioni + +| Comando | Valutazione | +|---|---| +| `session archive` | **PL**: usato dalla gestione sessioni del backend. | +| `session check` | **WF**: gate oggettivo della fase 5; verifica decisioni e schema linking. | +| `session close` | **PL**: usato dal backend. | +| `session delete` | **PL**: usato dal backend con i relativi controlli applicativi. | +| `session documents` | **WF/PL**: ricostruisce il contesto persistito e alimenta sia Pi sia la GUI. | +| `session fail` | **PL**: usato dal backend per rappresentare il fallimento terminale. | +| `session finalize` | **WF**: chiusura deterministica della fase finale e indicizzazione della domanda risolta. | +| `session list` | **PL**: alimenta la lista sessioni della GUI. | +| `session migrate` | **PL**: eseguito dal servizio one-shot di migrazione server; resta interno dietro `tht sessions migrate`. | +| `session new` | **WF/PL**: crea la persistenza iniziale della domanda; il backend dipende dal JSON restituito. | +| `session preferences get` | **PL**: lettura delle preferenze applicative. Interno. | +| `session preferences set` | **PL**: scrittura delle preferenze applicative. Interno. | +| `session reopen` | **PL**: riapertura dello stato terminale esposta dalla gestione sessioni. | +| `session retrieval-pack` | **WF**: legge il retrieval pack già persistito per il kickoff di Pi. Distinto da `search pack`, che lo costruisce. | +| `session set-group` | **PL**: rinomina il raggruppamento dalla GUI. | +| `session set-name` | **PL**: rinomina la sessione dalla GUI. | +| `session set-question` | **WF**: persiste deterministicamente domanda riscritta e assunzioni. Solo gate. | +| `session set-schema-linking` | **WF**: valida e scrive `schema_linking.json`. Solo gate. | +| `session show` | **WF/PL**: fonte compatta dello stato persistito per resume, gate e backend. | +| `session sync-schema-linking` | **WF**: riproietta deterministicamente il ledger nello schema linking. | +| `session unarchive` | **PL**: usato dalla gestione sessioni del backend. | + +### Schema e retrieval + +| Comando | Valutazione | +|---|---| +| `schema check` | **PL**: validazione delle annotazioni curate nel workflow workspace. | +| `schema columns` | **WF**: il gate usa il catalogo colonne per validare e correggere il linking. | +| `schema introspect` | **WF**: fallback previsto dal contratto quando manca lo schema fisico; la modalità refresh resta manutenzione. | +| `schema render` | **WF**: produce il contesto mschema usato dal modello. | +| `schema suggest-fks` | **PL**: comando del flusso operatore per le annotazioni FK curate. | +| `search find` | **WF**: ricerca mirata di evidence, valori e formule durante le fasi. | +| `search pack` | **WF/PL**: costruisce e persiste il contesto iniziale F1; usato anche dal backend. | + +### CTE, SQL e datamart + +| Comando | Valutazione | +|---|---| +| `cte info` | **WF**: restituisce SQL persistito, posizione nel piano e ultimo test. | +| `cte list` | **RIMUOVI**: nessun chiamante o test; `cte plan`, `cte info` e `session documents` coprono il bisogno. | +| `cte next` | **WF**: il gate determina il prossimo CTE da revisionare. | +| `cte plan` | **WF**: persiste l'ordine completo dei CTE. | +| `cte save` | **WF**: tool deterministico di scrittura usato dal gate. | +| `cte test` | **WF**: verifica read-only dei CTE prevista esplicitamente dal contratto. | +| `sql validate` | **WF**: validazione strutturale e read-only prima dell'esecuzione. | +| `sql preview` | **WF/PL**: preview controllata usata dal modello e dalla GUI. | +| `sql set-final` | **WF**: unica scrittura canonica di `sql_final.sql` attraverso il repository di sessione. | +| `sql export` | **PL**: esportazione richiesta dalla GUI. | +| `sql explain` | **RIMUOVI**: nessun chiamante, test o requisito nel workflow corrente. Si reintroduce solo con un vero passo di analisi del piano. | +| `sql save` | **RIMUOVI**: duplica `set-final` ed `export` e permette un percorso di scrittura non usato. | +| `datamart generate` | **WF**: fase 8 del workflow. | + +### Memory + +| Comando | Valutazione | +|---|---| +| `memory promote` | **WF**: preview dei candidati di promozione usata dal gate. | +| `memory save-one` | **WF**: persistenza atomica della singola memory approvata. | +| `memory search` | **WF**: recupero delle memory riutilizzabili nella fase 2. | +| `memory solved-index` | **WF**: recupero manuale previsto se l'indicizzazione al finalize fallisce. | +| `memory solved-search` | **WF**: recupero di domande risolte simili nelle fasi successive. | +| `memory list` | **ADV**: mantenere per amministrare record errati, ma fuori dall'help base. | +| `memory show` | **ADV**: mantenere insieme a `list` per ispezione puntuale. | +| `memory update` | **ADV**: mantenere per correggere il merito di una memory senza alterarne la provenienza. | +| `memory delete` | **ADV**: mantenere come rimedio selettivo; richiede conferma esplicita nella nuova CLI. | +| `memory index` | **ADV**: utile come riparazione/full-resync, ma va presentato come manutenzione e non come uso normale. | +| `memory clear` | **RIMUOVI**: distruzione globale non usata; confligge con una UX sicura di backup/ripristino. | +| `memory migrate` | **RIMUOVI**: migrazione legacy una tantum senza dati di produzione da preservare. | + +### Preprocessing, evidence e indici + +| Comando | Valutazione | +|---|---| +| `preprocess dwh` | **PL**: pipeline canonica usata da `tht workspace preprocess dwh/run`. | +| `preprocess evidence` | **PL**: pipeline canonica usata da `tht workspace preprocess evidence/run`. | +| `vector index-schema` | **PL**: indicizzazione schema usata dal workflow workspace. | +| `evidence extract` | **RIMUOVI**: primitivo superato dalla pipeline versionata `preprocess evidence`. Conservare soltanto la logica riusata. | +| `evidence index` | **RIMUOVI**: primitivo superato dalla stessa pipeline versionata. | +| `lsh build` | **RIMUOVI** come comando: è già uno step di `preprocess dwh`; il builder resta interno. | +| `lsh query` | **RIMUOVI**: probe visuale senza chiamanti, test o documentazione operativa. La ricerca applicativa passa da `search find`. | +| `vector init` | **RIMUOVI**: il controllo di Qdrant/embedder è ormai coperto dal reconciler di collezione, da `ollama ensure` e dal nuovo `tht doctor`. | + +### Formule di concetto + +| Comando | Valutazione | +|---|---| +| `formula save` | **RIMUOVI** dalla CLI corrente: nessun chiamante, test o flusso di approvazione lo usa. Conservare il formato/store e la lettura tramite `search find --kind formula`. | +| `formula list` | **RIMUOVI**: stesso sottosistema incompleto. Un futuro flusso di curation dovrà progettare insieme creazione, approvazione, elenco e modifica. | + +## Conseguenza per la nuova CLI unica + +La semplificazione migliore non consiste nel rinominare tutti i 55 contratti vivi o nel mostrarli +all'utente. Consiste nel mantenere un unico eseguibile `tht` con due livelli di visibilità: + +1. l'help ordinario mostra soltanto setup, lifecycle, backup/restore, Pi e workspace; +2. i contratti WF/PL restano invocabili dallo stesso eseguibile, ma sono interni/nascosti e usati da + backend, gate e job one-shot. + +In questo modo l'utente vede una CLI piccola, mentre il workflow non subisce una riscrittura inutile +e rischiosa. Non serve un secondo eseguibile né un alias `thothctl`. diff --git a/docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md b/docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md new file mode 100644 index 00000000..9f2095d9 --- /dev/null +++ b/docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md @@ -0,0 +1,148 @@ +# Proposta maintain-erase-enhance per i comandi `tht` + +Data: 2026-08-15 + +## Criterio + +- **MAINTAIN**: il comando resta disponibile senza modifiche sostanziali. Come richiesto, non viene + aggiunta una motivazione. +- **ERASE**: il comando viene eliminato dalla nuova CLI; la motivazione indica la duplicazione, il + superamento o l'assenza di un utilizzo reale. +- **ENHANCE**: la capacità viene mantenuta, ma il comando viene migliorato, accorpato o reso più + sicuro. La proposta indica l'intervento. + +La proposta copre tutti i 77 comandi terminali dell'attuale CLI Python. + +## Sintesi + +| Proposta | Numero | +|---|---:| +| MAINTAIN | 55 | +| ENHANCE | 8 | +| ERASE | 14 | +| **Totale** | **77** | + +## Lista completa + +### Diagnostica, configurazione e dipendenze + +| Comando | Proposta | +|---|---| +| `config check` | **ENHANCE** — incorporare la validazione nel comando pubblico `tht doctor`, mantenendo una funzione interna riutilizzabile e l'output strutturato. Evita due preflight sovrapposti. | +| `doctor` | **ENHANCE** — farne l'unica diagnostica multilivello: installazione, descriptor, Compose, storage, configurazione runtime, DWH, Pi, Qdrant ed embedder. Deve offrire output umano e `--json`, senza mutare lo stato. | +| `db ping` | **MAINTAIN** | +| `db fetch-ca` | **ENHANCE** — integrarlo nel setup guidato del workspace, mostrando endpoint e fingerprint prima della conferma. Può restare disponibile come operazione TLS avanzata, ma non come passaggio manuale obbligatorio. | +| `ollama ensure` | **MAINTAIN** | + +### Fasi e decision ledger + +| Comando | Proposta | +|---|---| +| `phase advance` | **MAINTAIN** | +| `phase meta` | **MAINTAIN** | +| `phase reopen` | **MAINTAIN** | +| `phase show` | **MAINTAIN** | +| `decision add` | **MAINTAIN** | +| `decision add-batch` | **MAINTAIN** | +| `decision add-join-set` | **MAINTAIN** | +| `decision list` | **ERASE** — non ha chiamanti reali e duplica il ledger già restituito da `session show --json`. | +| `decision retract` | **ERASE** — non è invocato dal workflow corrente; `phase reopen` è il percorso supportato per correggere e invalidare deterministicamente le decisioni. La semantica tombstone può restare nel dominio. | + +### Sessioni + +| Comando | Proposta | +|---|---| +| `session archive` | **MAINTAIN** | +| `session check` | **MAINTAIN** | +| `session close` | **MAINTAIN** | +| `session delete` | **MAINTAIN** | +| `session documents` | **MAINTAIN** | +| `session fail` | **MAINTAIN** | +| `session finalize` | **MAINTAIN** | +| `session list` | **MAINTAIN** | +| `session migrate` | **MAINTAIN** | +| `session new` | **MAINTAIN** | +| `session preferences get` | **MAINTAIN** | +| `session preferences set` | **MAINTAIN** | +| `session reopen` | **MAINTAIN** | +| `session retrieval-pack` | **MAINTAIN** | +| `session set-group` | **MAINTAIN** | +| `session set-name` | **MAINTAIN** | +| `session set-question` | **MAINTAIN** | +| `session set-schema-linking` | **MAINTAIN** | +| `session show` | **MAINTAIN** | +| `session sync-schema-linking` | **MAINTAIN** | +| `session unarchive` | **MAINTAIN** | + +### Schema e retrieval + +| Comando | Proposta | +|---|---| +| `schema check` | **MAINTAIN** | +| `schema columns` | **MAINTAIN** | +| `schema introspect` | **MAINTAIN** | +| `schema render` | **MAINTAIN** | +| `schema suggest-fks` | **MAINTAIN** | +| `search find` | **MAINTAIN** | +| `search pack` | **MAINTAIN** | + +### CTE, SQL e datamart + +| Comando | Proposta | +|---|---| +| `cte info` | **MAINTAIN** | +| `cte list` | **ERASE** — non ha chiamanti o test e sovrappone informazioni già disponibili con `cte plan`, `cte info` e `session documents`. | +| `cte next` | **MAINTAIN** | +| `cte plan` | **MAINTAIN** | +| `cte save` | **MAINTAIN** | +| `cte test` | **MAINTAIN** | +| `sql validate` | **MAINTAIN** | +| `sql preview` | **MAINTAIN** | +| `sql set-final` | **MAINTAIN** | +| `sql export` | **MAINTAIN** | +| `sql explain` | **ERASE** — non è usato né testato dal workflow attuale. Va reintrodotto soltanto se l'analisi del piano diventa un passo esplicito del processo. | +| `sql save` | **ERASE** — duplica `sql set-final` e `sql export` e introduce un percorso di scrittura non utilizzato. | +| `datamart generate` | **MAINTAIN** | + +### Memory + +| Comando | Proposta | +|---|---| +| `memory promote` | **MAINTAIN** | +| `memory save-one` | **MAINTAIN** | +| `memory search` | **MAINTAIN** | +| `memory solved-index` | **MAINTAIN** | +| `memory solved-search` | **MAINTAIN** | +| `memory list` | **ENHANCE** — trasformarlo in una vista amministrativa paginata, con filtri, provenienza, stato e output `--json`; non mostrarlo nell'help base. | +| `memory show` | **ENHANCE** — mostrare provenienza immutabile, decisione sorgente, stato dell'indice e riferimenti necessari a una correzione consapevole. | +| `memory update` | **ENHANCE** — limitare l'aggiornamento ai campi modificabili, mostrare un diff prima della conferma e impedire modifiche alla provenienza. | +| `memory delete` | **ENHANCE** — richiedere identificatore esatto e conferma esplicita, mostrare l'impatto e verificare la rimozione coerente da registro e indice. | +| `memory index` | **ENHANCE** — riposizionarlo come comando di repair: prima rileva il drift, poi ricostruisce soltanto con conferma e verifica finale. Non deve sembrare un'operazione ordinaria. | +| `memory clear` | **ERASE** — cancellazione globale non usata e troppo facile da eseguire per errore; backup/ripristino e cancellazione selettiva sono percorsi più sicuri. | +| `memory migrate` | **ERASE** — migrazione legacy una tantum; non esistono dati di produzione da preservare e la nuova architettura può partire direttamente dal formato corrente. | + +### Preprocessing, evidence e indici + +| Comando | Proposta | +|---|---| +| `preprocess dwh` | **MAINTAIN** | +| `preprocess evidence` | **MAINTAIN** | +| `vector index-schema` | **MAINTAIN** | +| `evidence extract` | **ERASE** — è un primitivo superato dalla pipeline versionata `preprocess evidence`; l'eventuale logica condivisa resta interna. | +| `evidence index` | **ERASE** — è un secondo primitivo superato dalla stessa pipeline, che già gestisce materializzazione, indicizzazione, versionamento e resume. | +| `lsh build` | **ERASE** — la costruzione LSH è già uno step di `preprocess dwh`; mantenere due ingressi permette esecuzioni parziali incoerenti. | +| `lsh query` | **ERASE** — probe visuale senza chiamanti, test o documentazione operativa; il workflow usa `search find`. | +| `vector init` | **ERASE** — il controllo di Qdrant ed embedder è già coperto dal reconciler della collezione, da `ollama ensure` e dal nuovo `tht doctor`. | + +### Formule di concetto + +| Comando | Proposta | +|---|---| +| `formula save` | **ERASE** — non ha chiamanti, test o un flusso di approvazione completo. Il formato e lo store possono restare disponibili alla ricerca finché non viene progettata una vera curation. | +| `formula list` | **ERASE** — appartiene allo stesso sottosistema incompleto; un futuro flusso deve progettare insieme creazione, approvazione, elenco, modifica e cancellazione. | + +## Impatto sulla UX + +I 55 comandi `MAINTAIN` comprendono molti contratti macchina intoccabili. Mantenerli non implica +mostrarli tutti nell'help principale. La futura CLI unica può conservare gli stessi percorsi per +backend, gate e job, mostrando all'utente soltanto i gruppi operativi di primo livello. diff --git a/docs/superpowers/plans/2026-08-14-thothctl-discovery-and-pi-update.md b/docs/superpowers/plans/2026-08-14-thothctl-discovery-and-pi-update.md new file mode 100644 index 00000000..938eff0c --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-thothctl-discovery-and-pi-update.md @@ -0,0 +1,86 @@ +# Simplified `thothctl` installation selection and Pi update Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Allow all existing `thothctl` commands to discover the installation descriptor automatically and make `thothctl pi update` use the checkout's pinned Pi version by default. + +**Architecture:** Add a small config-level resolver that chooses one validated installation descriptor from an explicit flag, environment variable, or bounded upward search from the working directory. Keep the existing Pi lifecycle transaction intact; resolve only the requested version/source at the CLI boundary so the update engine retains its safety and recovery guarantees. + +**Tech Stack:** Go 1.26, Docker Compose v2, existing `tools/thothctl` config and Pi lifecycle packages, Go tests. + +**Spec:** `docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md` + +## Global Constraints + +- Preserve every existing `pi` subcommand, alias, safety check, and explicit invocation form. +- `--installation` remains an explicit override and accepts only an absolute descriptor path named `thothii-installation.yaml`. +- Automatic discovery must not recursively scan `.artifacts`, home directories, or unrelated descendants. +- The default Pi version is the single `ARG PI_VERSION=` in the selected project's `docker/core.Dockerfile`. +- The default Pi update uses the existing transactional build path and must not install an arbitrary network “latest”. +- All failures remain sanitized and must not reveal secret values. + +## File Map + +- Create `tools/thothctl/internal/config/discovery.go` and `discovery_test.go` for bounded descriptor resolution and safe diagnostics. +- Modify `tools/thothctl/cmd/thothctl/main.go` and `main_test.go` for optional global selection, `pi update` defaults, help text, and dispatch. +- Create `tools/thothctl/internal/pi/version.go` and `version_test.go` for reading the project Pi pin. +- Modify `tools/thothctl/internal/pi/update.go` and `update_test.go` only if the default request needs a typed source/confirmation adjustment; keep lifecycle internals unchanged otherwise. +- Modify `docs/contracts/thothctl-pi.md`, `docs/install/pi-management.md`, and relevant command-contract verification scripts. + +### Task 1: Add bounded installation descriptor discovery + +**Files:** +- Create: `tools/thothctl/internal/config/discovery.go` +- Test: `tools/thothctl/internal/config/discovery_test.go` + +**Interface:** `func Resolve(explicit string, environment func(string) string, workingDirectory string) (string, error)`. + +- [x] Write failing tests for explicit-path precedence, `THOTHII_INSTALLATION`, `deploy/*/thothii-installation.yaml` discovery, parent discovery, `.artifacts` exclusion, invalid candidates, ambiguity, and no-candidate errors. +- [x] Run `cd tools/thothctl && go test ./internal/config -run 'TestResolve' -count=1`; confirm RED because `Resolve` is absent. +- [x] Implement a bounded upward walk. At each level inspect only the exact descriptor and immediate `deploy/*/thothii-installation.yaml` entries; skip `.artifacts`; require regular files; validate candidates through `config.Load`; deduplicate canonical paths; fail clearly on zero or multiple valid candidates. +- [x] Re-run the focused tests and confirm GREEN. +- [x] Refactor only after green, keeping path collection separate from candidate validation. + +### Task 2: Make the global installation option optional + +**Files:** +- Modify: `tools/thothctl/cmd/thothctl/main.go` +- Test: `tools/thothctl/cmd/thothctl/main_test.go` + +- [x] Add failing CLI tests proving `thothctl pi status` works from a project tree, the environment variable is used, an explicit flag wins, ambiguity fails before Docker, and all existing commands retain their dispatch. +- [x] Run the focused CLI tests and confirm RED because the current parser requires `--installation`. +- [x] Parse optional `--installation`, call `config.Resolve` with `THOTHII_INSTALLATION` and the process working directory, and update help to `thothctl [--installation PATH] `. +- [x] Run `cd tools/thothctl && go test ./cmd/thothctl -count=1`; confirm GREEN. + +### Task 3: Default `pi update` to the repository Pi pin + +**Files:** +- Create: `tools/thothctl/internal/pi/version.go` +- Test: `tools/thothctl/internal/pi/version_test.go` +- Modify: `tools/thothctl/cmd/thothctl/main.go` +- Test: `tools/thothctl/cmd/thothctl/main_test.go` + +**Interface:** `func ReadPinnedVersion(projectDirectory string) (string, error)`. + +- [x] Add failing tests for one valid Dockerfile pin, missing Dockerfile, duplicate default pins, malformed versions, and `pi update` without `--version`; retain explicit version and advanced pull tests. +- [x] Run `cd tools/thothctl && go test ./internal/pi ./cmd/thothctl -run 'Test(ReadPinnedVersion|ParsePiUpdate|RunPiUpdate)' -count=1`; confirm RED. +- [x] Read only `docker/core.Dockerfile`, require one default `ARG PI_VERSION=...`, validate it with the existing version grammar, and make the short request select build mode while preserving the lifecycle transaction. +- [x] Re-run focused tests and confirm GREEN. + +### Task 4: Update contracts without removing commands + +**Files:** +- Modify: `docs/contracts/thothctl-pi.md` +- Modify: `docs/install/pi-management.md` +- Modify: the documentation verification script that asserts the old mandatory update invocation. + +- [x] Document automatic descriptor discovery, the explicit override, `thothctl pi update` as the normal path, `--version` as an explicit pin, and the advanced pull/digest form. +- [x] Keep status, doctor, test/check, configure, restart, rollback, maintenance, and logs documented. +- [x] Run the targeted documentation checks and `git diff --check`. + +### Task 5: Full verification + +- [x] Run `cd tools/thothctl && go test ./... -count=1`. +- [x] Run `cd tools/thothctl && go build ./cmd/thothctl`. +- [x] Run the relevant documentation contract script and inspect `git status --short`. +- [x] Verify the help text contains the optional form and all existing commands; do not mutate the live Docker installation unless separately requested. diff --git a/docs/superpowers/plans/2026-08-15-unified-tht-cli-product-step.md b/docs/superpowers/plans/2026-08-15-unified-tht-cli-product-step.md new file mode 100644 index 00000000..08f96aeb --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-unified-tht-cli-product-step.md @@ -0,0 +1,1159 @@ +# Unified `tht` CLI Product Step Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Turn the repository's fragmented operator experience into one installable `tht` command that configures, builds, starts, updates, diagnoses, backs up, restores, and manages Pi, while preserving the indispensable NL-to-SQL workflow commands and simplifying the remaining command surface according to the approved maintain/erase/enhance audit. + +**Architecture:** The host-facing command is the existing native Go operator CLI, renamed from `thothctl` to `tht` and installed on the operating-system `PATH`. It discovers the current ThothII repository or Git worktree and its installation descriptor automatically. The Python workflow CLI remains named `tht` inside the `core` container and continues to own sessions, decisions, documents, SQL, and persistence. Host operations call Docker Compose directly or, when workflow checks are needed, invoke the container-local Python CLI. There is no compatibility alias, wrapper, second public command, host Python virtual environment, or requirement to build the CLI manually. + +**Tech Stack:** Go standard library, Docker Compose, Python/Typer, pytest, React 18, TypeScript, Vite, Vitest, Testing Library, Playwright, POSIX shell, PowerShell. + +**Spec:** `docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md` + +## Global Constraints + +- [ ] Expose exactly one public product command: `tht`. Remove `thothctl` rather than retaining an alias, wrapper, or deprecation period. +- [ ] Keep the Python workflow executable named `tht` inside `core`; do not introduce `tht-runtime` or a public `runtime` namespace. +- [ ] Preserve all 55 commands classified **MAINTAIN**, implement the 8 approved **ENHANCE** outcomes, and remove the 14 commands classified **ERASE**. The two audit reports are normative inputs. +- [ ] Make `--installation` optional on host commands. Explicit paths always win; otherwise discover the descriptor from the repository/worktree and then from the documented installation registry. +- [ ] Make `tht` callable from a project root or Git worktree root without `./`, `./bin/`, `~/bin/`, a shell wrapper, a Go build, or activation of a Python virtual environment. +- [ ] Make `tht setup` perform configuration, image build, container start, health verification, and diagnostics by default. `--configure-only` is the explicit opt-out before build/start. +- [ ] Make `tht pi update` resolve the latest stable Pi version when `--version` is omitted. A failed lookup must stop before any mutation; it must not silently reuse the Dockerfile pin. +- [ ] Preserve the user's current GLM 5.3 changes in `deploy/pi/models.json` and `deploy/pi/settings.json`; never replace those files with stale fixtures. +- [ ] Treat secrets as secret references by default. Do not print secret values, write them into tracked files, or include them in a backup unless the operator explicitly supplies both `--include-secrets` and `--yes`. +- [ ] Preserve unrelated dirty worktree changes and `.playwright-cli/`. Stage only files named by the current task. +- [ ] Keep frontend strings in English. Documentation may explain concepts in prose but all shown commands must be directly executable. +- [ ] Work test-first for every behavior change: add or tighten a failing test, run it to confirm the expected failure, implement the smallest complete change, rerun the focused test, then run the relevant suite. +- [ ] Do not deploy to the live Mac or restart port 8080 until all code, documentation, and automated gates pass. + +## Approved Command-Surface Baseline + +The implementation must end with this host-facing surface: + +```text +tht setup [--configure-only] [--installation PATH] +tht version +tht start [--build] [--installation PATH] +tht stop [--installation PATH] +tht status [--installation PATH] +tht doctor [--json] [--installation PATH] +tht logs [SERVICE] [--installation PATH] +tht update [--check-only] [--yes] [--drain] [--installation PATH] +tht backup [--output PATH] [--include-secrets --yes] [--drain] [--installation PATH] +tht restore ARCHIVE --yes [--drain] [--installation PATH] +tht sessions migrate --yes [--installation PATH] +tht remove [--yes ID...] [--installation PATH] +tht pi ... +tht workspace ... +``` + +`tht workspace ...` preserves every currently implemented native workspace operation, including +the existing inspect, preprocessing, schema, evidence, index, and vector operations. This plan does +not invent a second workspace-management surface merely to make the help tree look symmetrical. + +The Python workflow command surface inside `core` is governed by the approved audit: + +- **MAINTAIN:** 55 commands remain behaviorally and contractually available. +- **ENHANCE:** `config check`, `doctor`, `db fetch-ca`, `memory list`, `memory show`, `memory update`, `memory delete`, and `memory index` are improved or consolidated as described in Task 12. +- **ERASE:** `decision list`, `decision retract`, `cte list`, `sql explain`, `sql save`, `memory clear`, `memory migrate`, `evidence extract`, `evidence index`, `lsh build`, `lsh query`, `vector init`, `formula save`, and `formula list` are removed in Task 11. + +--- + +## Task 1: Rename the Native Operator CLI and Its Build Artifacts + +**Files:** + +- Rename: `tools/thothctl/` → `tools/tht/` +- Rename: `tools/tht/cmd/thothctl/` → `tools/tht/cmd/tht/` +- Rename: `docker/thothctl.Dockerfile` → `docker/tht.Dockerfile` +- Rename: `scripts/build-thothctl.sh` → `scripts/build-tht.sh` +- Rename/update: existing `scripts/test-thothctl-*.sh` files → corresponding `scripts/test-tht-*.sh` files +- Modify: Go module/import paths and package references under `tools/tht/` +- Modify: `.gitignore` + +### Steps + +- [ ] Add a command-identity test in `tools/tht/cmd/tht/main_test.go` that invokes the existing `run` entry point, requires the help banner and error prefix to use `tht`, exercises the `version` path, and proves `thothctl` is not an alias. +- [ ] Run the focused test before renaming and confirm it fails because the current executable and root command are still `thothctl`. + +```bash +cd tools/thothctl +go test ./cmd/thothctl -run 'TestRootCommandIdentity' -count=1 +``` + +- [ ] Rename the directory, command package, Dockerfile, build script, smoke scripts, binary outputs, archive names, and image labels. Update imports mechanically, including the Go module path if it contains `/tools/thothctl`. +- [ ] Make `scripts/build-tht.sh` emit only `tht` binaries and archives such as `tht-darwin-arm64`, never `thothctl-*`. +- [ ] Remove all executable aliases and wrapper generation. Historical design documents may retain the old name as history; active source, tests, packaging, and user documentation may not. +- [ ] Run the renamed test and all Go tests. + +```bash +cd tools/tht +go test ./cmd/tht -run 'TestRootCommandIdentity' -count=1 +go test ./... +``` + +- [ ] Run a scoped stale-name scan over the renamed implementation and build assets. Active documentation is intentionally updated in Task 14; do not partially rewrite it here. + +```bash +rg -n 'thothctl|THOTHCTL' tools/tht docker scripts \ + -g '!scripts/test-tht-command-docs.sh' +``` + +- [ ] Commit only the rename and mechanical identity changes. + +```bash +git add -A -- tools/thothctl tools/tht docker/thothctl.Dockerfile docker/tht.Dockerfile \ + scripts/build-thothctl.sh scripts/build-tht.sh \ + scripts/test-thothctl-build-contract.sh scripts/test-tht-build-contract.sh \ + scripts/thothctl-update-smoke.sh scripts/tht-update-smoke.sh .gitignore +git commit -m "refactor(cli): rename operator command to tht" +``` + +--- + +## Task 2: Add Cross-Platform `tht` Installers + +**Files:** + +- Create: `scripts/install-tht.sh` +- Create: `scripts/install-tht.ps1` +- Create: `scripts/test-install-tht.sh` +- Create: `scripts/test-install-tht.ps1` +- Modify: `scripts/build-tht.sh` + +### Installer contract + +- macOS/Linux: use the repository-pinned Docker builder to produce the native binary for the current OS/architecture, install it as `/usr/local/bin/tht` by default, use elevation only for the final atomic install when required, and verify that the installed command is resolvable on `PATH`. +- Windows: install `tht.exe` under `%LOCALAPPDATA%\ThothII\bin`, add that directory to the current user's `PATH` when absent, and explain when a new terminal is required. +- Tests may override the destination with `THT_INSTALL_DIRECTORY`; production users are not instructed to set this variable. +- Re-running the installer replaces only the installed `tht` binary atomically and leaves installation data untouched. + +### Steps + +- [ ] Write shell installer tests that use a temporary `THT_INSTALL_DIRECTORY`, a fake build artifact, and a controlled `PATH`. Assert executable permissions, atomic replacement, `tht version`, and idempotency. +- [ ] Write PowerShell tests for the equivalent Windows behavior, including paths containing spaces and a pre-existing user `PATH` entry. +- [ ] Run both tests and confirm failure because the installers do not exist. + +```bash +bash scripts/test-install-tht.sh +pwsh -NoProfile -File scripts/test-install-tht.ps1 +``` + +- [ ] Implement `scripts/install-tht.sh` with explicit OS/architecture detection, a temporary staging directory, checksum validation when using a packaged artifact, and an atomic final rename. +- [ ] Implement `scripts/install-tht.ps1` with the same contract and user-level `PATH` update. +- [ ] Make both installers invoke `scripts/build-tht.sh` internally, which uses the repository-pinned Docker builder. The user must never install Go, select an artifact, or invoke a Go compiler. +- [ ] Rerun focused tests and package builds for Darwin arm64/amd64, Linux arm64/amd64, and Windows amd64. + +```bash +bash scripts/test-install-tht.sh +pwsh -NoProfile -File scripts/test-install-tht.ps1 +bash scripts/build-tht.sh --all +``` + +- [ ] Commit the installer slice. + +```bash +git add scripts/install-tht.sh scripts/install-tht.ps1 scripts/test-install-tht.sh \ + scripts/test-install-tht.ps1 scripts/build-tht.sh +git commit -m "feat(cli): install tht as a system command" +``` + +--- + +## Task 3: Discover the Project, Worktree, and Installation Descriptor Automatically + +**Files:** + +- Create: `tools/tht/internal/project/discovery.go` +- Create: `tools/tht/internal/project/discovery_test.go` +- Modify: `tools/tht/internal/config/discovery.go` +- Modify: `tools/tht/internal/config/discovery_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package project + +type Root struct { + Path string + IsWorktree bool +} + +func Discover(start string) (Root, error) +``` + +The descriptor resolver keeps its existing explicit-path support and applies this precedence: + +1. `--installation PATH` supplied to the current command. +2. `THOTHII_INSTALLATION` when explicitly set by automation. +3. One valid `thothii-installation.yaml` in the current directory or its immediate `deploy/*` children. +4. The same bounded search while walking parent directories to the discovered repository/worktree root. +5. A concise actionable error suggesting `tht setup` when no descriptor exists, or listing bounded candidates and requiring optional `--installation` when more than one exists. + +### Steps + +- [ ] Add table-driven tests for invocation from the repository root, a nested directory, a linked Git worktree, one immediate `deploy//thothii-installation.yaml`, multiple immediate descriptors, a missing descriptor, `THOTHII_INSTALLATION`, and an explicit descriptor override. +- [ ] Add tests proving `tht`, `tht help`, `tht version`, and `tht setup` do not require a pre-existing descriptor. +- [ ] Run the focused tests and confirm the current resolver fails root/worktree and descriptor-free bootstrap cases. + +```bash +cd tools/tht +go test ./internal/project ./internal/config ./cmd/tht -run 'TestDiscover|TestResolve|TestBootstrapCommands' -count=1 +``` + +- [ ] Implement root detection using repository markers (`.git` file or directory, `compose.yaml`, `deploy/`, and the expected ThothII source layout). Resolve symlinks for identity while retaining the user's invocation path for messages. +- [ ] Extend descriptor discovery without changing explicit `--installation` semantics. Keep the search bounded to current/ancestor directories and immediate `deploy/*`; do not scan the home directory or fall back to `/Users/mp/thothii-installation.yaml`. +- [ ] Return ambiguity as an error listing installation IDs and descriptor paths without exposing secret values. +- [ ] Run all Go tests. Defer system-PATH smoke calls to Task 15 so the old Mac installation is not changed prematurely. + +```bash +cd tools/tht && go test ./... +``` + +- [ ] Commit discovery behavior. + +```bash +git add tools/tht/internal/project tools/tht/internal/config tools/tht/cmd/tht +git commit -m "feat(cli): discover ThothII projects and installations" +``` + +--- + +## Task 4: Generate Setup Files Safely + +**Files:** + +- Create: `tools/tht/internal/setup/request.go` +- Create: `tools/tht/internal/setup/files.go` +- Create: `tools/tht/internal/setup/files_test.go` +- Modify: `.gitignore` +- Modify: `deploy/env/local.env.example` +- Modify: `deploy/psd/thothii-installation.yaml.example` +- Modify: `deploy/psd/operator.env.example` +- Modify: `tools/tht/cmd/tht/main.go` + +### Interfaces + +```go +package setup + +type Request struct { + ProjectRoot string + InstallationID string + Profile string + ConfigureOnly bool + NonInteractive bool +} + +type FilesResult struct { + DescriptorPath string + EnvironmentPath string + Created []string +} + +func EnsureFiles(request Request, input io.Reader, output io.Writer) (FilesResult, error) +``` + +### Steps + +- [ ] Add tests for a fresh checkout, a linked worktree, existing compatible files, conflicting files, interrupted writes, paths with spaces, and secret prompts. Assert that tracked examples are never modified. +- [ ] Require generated files to live under `deploy//`, be ignored by Git except for tracked examples, and contain secret file references rather than secret values. +- [ ] Run the tests and confirm they fail because setup file generation is absent. + +```bash +cd tools/tht +go test ./internal/setup -run 'TestEnsureFiles' -count=1 +``` + +- [ ] Implement prompts for installation ID, deployment profile, externally reachable endpoints, workspace selection, and secret-file locations. Accept safe defaults in interactive mode and explicit flags/environment in non-interactive automation. +- [ ] Write `deploy//thothii-installation.yaml` and `deploy//operator.env` atomically with restrictive permissions where the platform supports them. +- [ ] Refuse to overwrite a conflicting descriptor or environment file. Report the exact file and corrective action. +- [ ] Create protected external secret-file templates only after explicit confirmation, never overwrite an existing secret file, and store only their paths in generated configuration. +- [ ] Rerun setup tests and verify the bounded discovery from Task 3 finds the generated descriptor and no generated file is tracked. + +```bash +cd tools/tht && go test ./internal/setup -count=1 +``` + +- [ ] Commit setup-file generation. + +```bash +git add tools/tht/internal/setup tools/tht/cmd/tht/main.go \ + deploy/env/local.env.example deploy/psd/thothii-installation.yaml.example \ + deploy/psd/operator.env.example .gitignore +git commit -m "feat(setup): generate local installation configuration" +``` + +--- + +## Task 5: Make `tht setup` Build, Start, and Verify the Product + +**Files:** + +- Create: `tools/tht/internal/setup/run.go` +- Create: `tools/tht/internal/setup/run_test.go` +- Modify: `tools/tht/internal/compose/runner.go` +- Modify: `tools/tht/internal/compose/runner_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package setup + +type Result struct { + DescriptorPath string + ProjectName string + Configured bool + Built bool + Started bool + Healthy bool +} + +func Run( + ctx context.Context, + runner compose.Runner, + request Request, + input io.Reader, + output io.Writer, +) (Result, error) +``` + +### Ordered setup workflow + +1. Discover and validate the repository/worktree. +2. Check Docker Engine, Docker Compose, supported architecture, and line-ending compatibility. +3. Generate or validate local configuration. +4. Run `docker compose config` against the resolved descriptor and profiles. +5. Stop successfully when `--configure-only` is set. +6. Build required images, including the `core` image containing Pi. +7. Start the stack with the installation-specific Compose project name. +8. Wait for frontend, core, qdrant, embedding, and one-shot model initialization health. +9. Run aggregate `tht doctor` and `tht pi doctor`. +10. Print the frontend URL and concise next actions. + +### Steps + +- [ ] Add runner-fake tests asserting the exact order above, immediate stop for `--configure-only`, failure propagation, retryable health polling, and cleanup messaging after partial startup. +- [ ] Add CLI tests proving `tht setup` defaults to build/start and that only `--configure-only` disables those phases. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/setup ./cmd/tht -run 'TestRun|TestSetupCommand' -count=1 +``` + +- [ ] Implement orchestration using the existing descriptor/Compose abstractions. Do not duplicate command execution logic in the top-level argument dispatcher. +- [ ] Make health waits bounded and identify the failing service, last health state, and useful `tht logs ` command. +- [ ] Ensure setup can be rerun idempotently to repair/start an already configured checkout. +- [ ] Rerun focused and full Go suites. + +```bash +cd tools/tht +go test ./internal/setup ./internal/compose ./cmd/tht -count=1 +go test ./... +``` + +- [ ] Commit the complete setup workflow. + +```bash +git add tools/tht/internal/setup tools/tht/internal/compose tools/tht/cmd/tht +git commit -m "feat(setup): build start and verify ThothII" +``` + +--- + +## Task 6: Add `version`, Aggregate `doctor`, and `start --build` + +**Files:** + +- Create: `tools/tht/internal/version/info.go` +- Create: `tools/tht/internal/version/info_test.go` +- Create: `tools/tht/internal/doctor/report.go` +- Create: `tools/tht/internal/doctor/report_test.go` +- Create: `tools/tht/internal/service/service.go` +- Create: `tools/tht/internal/service/service_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package doctor + +type Check struct { + Name string `json:"name"` + Status string `json:"status"` + Detail string `json:"detail"` +} + +type Report struct { + OK bool `json:"ok"` + Checks []Check `json:"checks"` +} + +func Run(ctx context.Context, installation config.Installation, runner Runner) (Report, error) +``` + +### Required behavior + +- `tht version` works without an installation and reports CLI semantic version, commit, build time, OS, and architecture. When an installation is discoverable, it may additionally report the deployed product and Pi versions. +- `tht doctor` aggregates descriptor validation, Compose availability/configuration, file permissions, required volume presence, service health, frontend/core reachability, workspace registry validity, container-local workflow diagnostics, and Pi diagnostics. +- `tht doctor --json` writes one pristine JSON document to stdout; all progress and warnings go to stderr. +- `tht start` starts without rebuilding. `tht start --build` runs the required build before Compose up and then performs bounded health checks. + +### Steps + +- [ ] Add tests for descriptor-free `version`, deterministic JSON, unavailable Docker, stopped/running core, a failed workflow check, and a redacted secret path. +- [ ] Add service tests proving `--build` changes the runner sequence from `up` to `build → up → health`, while normal `start` remains `up → health`. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/version ./internal/doctor ./internal/service ./cmd/tht \ + -run 'TestVersion|TestDoctor|TestStart' -count=1 +``` + +- [ ] Implement build metadata with linker defaults that remain useful in local source builds. +- [ ] Implement aggregate diagnostics as typed checks. Invoke the Python workflow `tht doctor --json` only through `docker compose exec -T core ...` when `core` is running; never require a host virtual environment. +- [ ] Implement `start --build` through the shared Compose runner. +- [ ] Verify JSON output and full tests. + +```bash +cd tools/tht && go test ./... +go run ./cmd/tht version +go run ./cmd/tht doctor --json | jq -e '.ok != null and (.checks | type == "array")' +``` + +- [ ] Commit this operator-observability slice. + +```bash +git add tools/tht/internal/version tools/tht/internal/doctor tools/tht/internal/service tools/tht/cmd/tht +git commit -m "feat(cli): add version diagnostics and build-aware start" +``` + +--- + +## Task 7: Make `tht pi update` Resolve the Latest Stable Pi Version + +**Files:** + +- Create: `tools/tht/internal/pi/latest.go` +- Create: `tools/tht/internal/pi/latest_test.go` +- Modify: `tools/tht/internal/pi/update.go` +- Modify: `tools/tht/internal/pi/update_test.go` +- Modify: `tools/tht/internal/pi/commands.go` +- Modify: `tools/tht/cmd/tht/main.go` + +### Interfaces + +```go +package pi + +type RegistryClient interface { + LatestStable(ctx context.Context, packageName string) (string, error) +} + +func ResolveRequestedVersion( + ctx context.Context, + requested string, + packageName string, + registry RegistryClient, +) (string, error) +``` + +### Required behavior + +- `tht pi update` means “install the latest stable Pi release available from the package registry.” +- `tht pi update --version X.Y.Z` installs that explicit stable version and skips latest-version discovery. +- Prerelease versions require an explicit full version; latest discovery ignores prereleases. +- Failure, malformed registry data, timeout, or an empty version stops before drain, Dockerfile mutation, image build, container replacement, or configuration changes. +- The command keeps the existing `--source build|pull`, immutable image digest, `--yes`, `--drain`, rollback, status, doctor, test, logs, and configure capabilities. `--installation` remains optional through Task 3 discovery. + +### Steps + +- [ ] Add registry-client tests for a stable release, prerelease-only data, malformed JSON, timeout, package-not-found, and explicit version bypass. +- [ ] Change the update transaction test so an omitted version expects the resolved latest stable release rather than the current `ARG PI_VERSION` value from `docker/core.Dockerfile`. +- [ ] Add a no-mutation assertion around every discovery failure. +- [ ] Run focused tests and confirm the old default-to-Dockerfile behavior fails them. + +```bash +cd tools/tht +go test ./internal/pi ./cmd/tht -run 'TestResolveRequestedVersion|TestPiUpdate' -count=1 +``` + +- [ ] Implement a bounded HTTPS client for the registry used by the Pi package declared in `docker/core.Dockerfile`. Parse and validate strict semantic versions; do not shell out to a globally installed npm executable. +- [ ] Resolve the final version before acquiring a drain or update lock. Feed the resolved version into the existing transactional build/pull and rollback path. +- [ ] Leave the project release pin in `docker/core.Dockerfile` unchanged as the reproducible clean-build default. Record the selected newer image/version in installation update state so ordinary restart does not revert it; do not use or rewrite the pin as the meaning of “latest.” +- [ ] Rerun all Pi and Go tests, including explicit build and immutable pull paths. + +```bash +cd tools/tht +go test ./internal/pi -count=1 +go test ./... +``` + +- [ ] Commit latest-version resolution without altering the user's GLM files. + +```bash +git add tools/tht/internal/pi tools/tht/cmd/tht/main.go +git commit -m "feat(pi): update to the latest stable release by default" +``` + +--- + +## Task 8: Implement Transactional Installation Backups + +**Files:** + +- Create: `tools/tht/internal/backup/manifest.go` +- Create: `tools/tht/internal/backup/manifest_test.go` +- Create: `tools/tht/internal/backup/create.go` +- Create: `tools/tht/internal/backup/create_test.go` +- Create: `tools/tht/internal/lifecycle/lock.go` +- Create: `tools/tht/internal/lifecycle/lock_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package backup + +type Manifest struct { + SchemaVersion int `json:"schema_version"` + InstallationID string `json:"installation_id"` + CreatedAt time.Time `json:"created_at"` + SourceRevision string `json:"source_revision"` + IncludesSecrets bool `json:"includes_secrets"` + Entries []Entry `json:"entries"` +} + +type CreateRequest struct { + Output string + IncludeSecrets bool + Confirm bool + Drain bool +} + +func Create(ctx context.Context, installation config.Installation, request CreateRequest) (Result, error) +``` + +### Backup contents + +- Effective installation descriptor and non-secret local environment/configuration files. +- Pi declarative host configuration, including `deploy/pi/models.json` and `deploy/pi/settings.json`. +- Named volumes: `settings`, `pi-state`, `workspace-registry`, `workspace-secrets`, `sessions`, `qdrant-data`, and `embedding-models`. +- Server preservation roots declared by the installation descriptor. +- A manifest containing checksums, logical ownership, source revision, image identities, Compose project name, volume metadata, and whether secret contents are present. + +The installation-owned `workspace-secrets` volume is part of the consistent volume snapshot. External secret-file contents referenced by the installation are excluded by default; their paths and digests are recorded so restore can verify that they still exist. Including those external files requires `--include-secrets --yes`, writes the archive with mode `0600` where supported, and emits a clear custody warning without printing values. + +### Steps + +- [ ] Add manifest tests for deterministic entry ordering, checksums, path normalization, secret markers, and schema-version validation. +- [ ] Add create tests for the seven required volumes, stopped and running installations, active sessions without `--drain`, explicit drain, custom output, default output, write failure, and cleanup of incomplete archives. +- [ ] Assert the default path is `~/.thothii/backups//` and the final archive name contains a UTC timestamp and source revision. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/backup ./internal/lifecycle ./cmd/tht \ + -run 'TestManifest|TestCreate|TestBackupCommand' -count=1 +``` + +- [ ] Implement a per-installation lifecycle lock shared by backup, restore, Pi update, and product update. +- [ ] Quiesce or stop mutable services before snapshotting. Refuse an unsafe live snapshot when active sessions exist and `--drain` was not supplied. +- [ ] Stream volume data into an archive through a minimal helper container; do not materialize secret data in the repository or process arguments. +- [ ] Write the manifest last, fsync the temporary archive, atomically rename it, and remove partial output on error. +- [ ] Run backup tests and inspect a fixture archive to verify that default backups contain no secret payloads. + +```bash +cd tools/tht && go test ./internal/backup ./internal/lifecycle -count=1 +go test ./... +``` + +- [ ] Commit backup support. + +```bash +git add tools/tht/internal/backup tools/tht/internal/lifecycle tools/tht/cmd/tht +git commit -m "feat(cli): add transactional installation backups" +``` + +--- + +## Task 9: Implement Validated Restore with a Recovery Checkpoint + +**Files:** + +- Create: `tools/tht/internal/backup/restore.go` +- Create: `tools/tht/internal/backup/restore_test.go` +- Create: `tools/tht/internal/backup/preflight.go` +- Create: `tools/tht/internal/backup/preflight_test.go` +- Modify: `tools/tht/internal/backup/create.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package backup + +type RestoreRequest struct { + Archive string + Confirm bool + Drain bool +} + +type RestoreResult struct { + Checkpoint string + Restarted bool + Verified bool +} + +func Restore( + ctx context.Context, + installation config.Installation, + request RestoreRequest, +) (RestoreResult, error) +``` + +### Restore contract + +Before modifying the installation, restore must validate the archive schema, every checksum, path traversal safety, installation identity, secret policy, required free disk space, target ownership/permissions, volume mapping, and image/config compatibility. It then creates a non-secret recovery checkpoint of the current installation, acquires the lifecycle lock, drains or refuses active sessions, restores into controlled targets, restarts the stack only when it was previously running, and runs health, aggregate doctor, Pi doctor, and workspace inspection. A failure after mutation leaves the target in a recoverable stopped state and prints the checkpoint path; it must not compound damage with an unrequested automatic restore. + +### Steps + +- [ ] Add adversarial archive tests for `../` traversal, absolute paths, symlink escapes, duplicate entries, checksum mismatch, unknown schema, wrong installation ID, secret-bearing archives without required protections, insufficient disk, and invalid volume ownership. +- [ ] Add transaction tests for success, failure before mutation, failure after one restored volume, failed restart, and failed health verification. Every post-mutation failure must stop the target, retain the checkpoint, and report a deterministic recovery command. +- [ ] Require positional archive plus `--yes`; do not allow an interactive typo to start restore without a complete preflight. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/backup ./cmd/tht -run 'TestPreflight|TestRestore' -count=1 +``` + +- [ ] Implement archive validation without extracting untrusted paths directly to final destinations. +- [ ] Create the rollback checkpoint through the same manifest/archive primitives as Task 8. +- [ ] Restore configuration and volumes in a deterministic order; retain the checkpoint path in both success and error messages. +- [ ] Verify with service health, `tht doctor`, `tht pi doctor`, and workspace inspection before declaring success. +- [ ] Rerun focused, package, and full Go tests. + +```bash +cd tools/tht +go test ./internal/backup -count=1 +go test ./... +``` + +- [ ] Commit restore support. + +```bash +git add tools/tht/internal/backup tools/tht/cmd/tht +git commit -m "feat(cli): add validated restore with rollback" +``` + +--- + +## Task 10: Turn `tht update` into a Full Product Update Transaction + +**Files:** + +- Create: `tools/tht/internal/productupdate/plan.go` +- Create: `tools/tht/internal/productupdate/plan_test.go` +- Create: `tools/tht/internal/productupdate/run.go` +- Create: `tools/tht/internal/productupdate/run_test.go` +- Modify: `tools/tht/internal/backup/create.go` +- Modify: `tools/tht/internal/lifecycle/lock.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package productupdate + +type Request struct { + CheckOnly bool + Confirm bool + Drain bool +} + +type Result struct { + PreviousImages map[string]string + CurrentImages map[string]string + Checkpoint string + RolledBack bool +} + +func Plan(ctx context.Context, installation config.Installation) (UpdatePlan, error) +func Run(ctx context.Context, installation config.Installation, request Request) (Result, error) +``` + +### Transaction contract + +- `tht update --check-only` validates compatibility and prints the exact image pull/build, migration, restart, and verification plan without mutating the Git checkout, containers, volumes, or configuration. +- `tht update --yes` updates the complete ThothII deployment according to the current checkout and installation descriptor: pull externally sourced images, build source-defined images, run required preflight/migrations, recreate changed services, and verify the complete product. +- Dirty source files are not reset, overwritten, committed, or pulled. Updating source from Git is deliberately outside this command; the operator chooses the checkout/revision and `tht update` deploys it. +- Before mutation, acquire the lifecycle lock, enforce active-session drain policy, and create a rollback checkpoint through Task 8. +- Record previous immutable image identities. A build/pull, migration, restart, health, aggregate-doctor, or Pi-doctor failure restores configuration/volumes as needed and recreates the previous images. + +### Steps + +- [ ] Add plan tests for a no-op installation, changed built image, changed pulled image, migration required, incompatible descriptor, dirty worktree, and unavailable registry. +- [ ] Add transaction tests for every failure boundary and assert rollback uses recorded image digests rather than mutable tags. +- [ ] Add CLI tests for `--check-only`, required `--yes` before mutation, optional `--drain`, and optional `--installation`. +- [ ] Run focused tests and confirm the current check-only implementation cannot execute the transaction. + +```bash +cd tools/tht +go test ./internal/productupdate ./cmd/tht \ + -run 'TestUpdatePlan|TestProductUpdate|TestUpdateCommand' -count=1 +``` + +- [ ] Implement pure planning first, then the transactional runner. Keep user confirmation outside the mutation core so tests can call it deterministically. +- [ ] Reuse Compose, lifecycle, backup, health, doctor, and Pi diagnostic components; do not introduce parallel shell orchestration. +- [ ] Make failure output state whether rollback completed and provide the retained checkpoint path. +- [ ] Rerun update, backup, Pi, and full Go suites. + +```bash +cd tools/tht +go test ./internal/productupdate ./internal/update ./internal/backup ./internal/pi -count=1 +go test ./... +``` + +- [ ] Commit the product-update transaction. + +```bash +git add tools/tht/internal/productupdate tools/tht/internal/backup \ + tools/tht/internal/lifecycle tools/tht/cmd/tht +git commit -m "feat(cli): update the full product transactionally" +``` + +--- + +## Task 11: Apply the 14 Approved **ERASE** Decisions to the Python Workflow CLI + +**Files:** + +- Create: `harness/tests/fixtures/approved_cli_surface.json` +- Create: `harness/tests/test_cli_surface.py` +- Modify: `harness/tht/cli/decision_cmd.py` +- Modify: `harness/tht/cli/cte_cmd.py` +- Modify: `harness/tht/cli/sql_cmd.py` +- Modify: `harness/tht/cli/memory_cmd.py` +- Modify: `harness/tht/cli/evidence_cmd.py` +- Modify: `harness/tht/cli/lsh_cmd.py` +- Modify: `harness/tht/cli/vector_cmd.py` +- Modify: `harness/tht/cli/formula_cmd.py` +- Modify: `harness/tests/integration/test_gate_cli_signatures.py` +- Delete or rewrite: tests dedicated exclusively to erased CLI entry points, including `harness/tests/test_decision_retract_cli.py` + +### Commands to erase + +```text +decision list +decision retract +cte list +sql explain +sql save +memory clear +memory migrate +evidence extract +evidence index +lsh build +lsh query +vector init +formula save +formula list +``` + +Removing a command means removing its Typer registration, help entry, CLI-only parsing code, CLI-only tests, and active documentation. Underlying domain functions may remain only when a maintained workflow, preprocessing pipeline, or test imports them directly. Delete dead implementation only after a repository-wide reachability check. + +### Steps + +- [ ] Build `approved_cli_surface.json` from the two approved audit reports. It must enumerate all 55 maintained paths and the 8 enhanced paths and explicitly blacklist the 14 erased paths. +- [ ] Add a recursive Typer help test that compares the actual command tree to this fixture. Add integration assertions that every command invoked by `harness/.pi/extensions/tht-gate.js`, `harness/.pi/skills/tht-sessione/SKILL.md`, backend `ThtRunner`, preprocessing jobs, and deployment smoke tests is still present. +- [ ] Run the command-surface tests before removal and confirm they fail because the 14 erased commands are still exposed. + +```bash +cd harness +.venv/bin/pytest -q tests/test_cli_surface.py tests/integration/test_gate_cli_signatures.py +``` + +- [ ] Remove the 14 registrations and CLI-only code. Preserve `phase reopen` as the supported correction/invalidation flow, `session show --json` as the ledger view, `sql set-final`/`sql export`, versioned `preprocess evidence`/`preprocess dwh`, collection reconciliation, `ollama ensure`, and `search find`. +- [ ] Delete or rewrite tests that assert the obsolete surface. Add negative tests requiring a nonzero exit and normal “no such command” message for every erased path. +- [ ] Use import and call-site scans before deleting any shared function. + +```bash +rg -n 'decision_retract|cte_list|sql_explain|sql_save|memory_clear|memory_migrate|evidence_(extract|index)|lsh_(build|query)|vector_init|formula_(save|list)' \ + harness backend frontend scripts docs +``` + +- [ ] Run the full non-L2 harness suite and the gate signature test. + +```bash +cd harness +.venv/bin/ruff check . +.venv/bin/pytest -q +``` + +- [ ] Commit the approved surface reduction. + +```bash +git add harness/tht/cli/decision_cmd.py harness/tht/cli/cte_cmd.py \ + harness/tht/cli/sql_cmd.py harness/tht/cli/memory_cmd.py \ + harness/tht/cli/evidence_cmd.py harness/tht/cli/lsh_cmd.py \ + harness/tht/cli/vector_cmd.py harness/tht/cli/formula_cmd.py \ + harness/tests/fixtures/approved_cli_surface.json harness/tests/test_cli_surface.py \ + harness/tests/integration/test_gate_cli_signatures.py +git add -u -- harness/tests/test_decision_retract_cli.py +git commit -m "refactor(cli): remove obsolete workflow commands" +``` + +--- + +## Task 12: Implement the 8 Approved **ENHANCE** Outcomes + +**Files:** + +- Modify: `harness/tht/cli/config_cmd.py` +- Modify: `harness/tht/cli/doctor_cmd.py` +- Modify: `harness/tht/cli/db_cmd.py` +- Modify: `harness/tht/cli/memory_cmd.py` +- Modify: `harness/tht/memory.py` +- Create: `harness/tests/test_cli_enhanced_surface.py` +- Modify: `harness/tests/test_doctor_cli.py` +- Modify: `harness/tests/test_cli_config_environment.py` +- Modify: `harness/tests/test_memory_metadata.py` +- Modify: `harness/tests/test_qdrant_cli_commands.py` +- Create: `tools/tht/internal/setup/tls.go` +- Create: `tools/tht/internal/setup/tls_test.go` +- Modify: `tools/tht/internal/setup/run.go` +- Modify: `tools/tht/internal/setup/run_test.go` + +### Exact enhanced outcomes + +1. `config check`: keep its validation as a reusable internal function and structured result, but make public `tht doctor` the normal single preflight. Avoid two competing operator diagnostics. +2. `doctor`: make it the non-mutating, multilayer diagnostic for installation, descriptor, Compose, storage, runtime configuration, DWH, Pi, Qdrant, and embedder, with readable output and pristine `--json`. +3. `db fetch-ca`: integrate CA retrieval into guided workspace setup. Show endpoint, certificate subject, validity, SHA-256 fingerprint, and destination before confirmation. Keep the standalone operation as an advanced TLS command, not a mandatory manual setup step. +4. `memory list`: make it an advanced paginated administrative view with filters for state, provenance, and indexing status plus `--json`; keep it out of concise/basic help. +5. `memory show`: display immutable provenance, source decision, mutable fields, index state, timestamps, and references needed for a safe correction. +6. `memory update`: permit only documented mutable fields, print a diff, require explicit confirmation, and reject attempts to alter identity or provenance. +7. `memory delete`: require an exact identifier and explicit confirmation, show registry/index impact, remove consistently from both, and verify absence afterward. +8. `memory index`: treat it as repair. Detect and report drift first; rebuild only with explicit confirmation; verify registry/index consistency afterward. + +### Steps + +- [ ] Add focused tests for every outcome above, including JSON purity, no mutation by doctor, TLS fingerprint confirmation, pagination bounds, provenance immutability, exact-ID deletion, drift-only repair, confirmation refusal, and partial index failure. +- [ ] Run the focused tests and confirm they fail against current behavior. + +```bash +cd harness +.venv/bin/pytest -q tests/test_cli_enhanced_surface.py tests/test_doctor_cli.py \ + tests/test_cli_config_environment.py tests/test_memory_metadata.py \ + tests/test_qdrant_cli_commands.py +``` + +- [ ] Extract configuration validation into a typed result consumed by both container-local doctor and the host aggregate doctor from Task 6. Keep `config check` callable for automation but mark it advanced in help. +- [ ] Extend doctor without adding repair side effects. A failing layer changes exit status and report data but never changes files, volumes, indexes, or containers. +- [ ] Integrate TLS CA fetch into host workspace create/update setup with a two-stage inspect/confirm flow. Reject hostname mismatch and invalid/expired certificates before writing. +- [ ] Implement memory list/show/update/delete/index with a shared repository/index transaction boundary and postcondition checks. +- [ ] Rerun focused tests, then the full harness suite and Go workspace tests. + +```bash +cd harness +.venv/bin/ruff check . +.venv/bin/pytest -q +cd ../tools/tht +go test ./internal/setup ./internal/doctor -count=1 +go test ./... +``` + +- [ ] Commit enhanced diagnostics, TLS setup, and memory administration. + +```bash +git add harness/tht/cli/config_cmd.py harness/tht/cli/doctor_cmd.py \ + harness/tht/cli/db_cmd.py harness/tht/cli/memory_cmd.py harness/tht/memory.py \ + harness/tests/test_cli_enhanced_surface.py harness/tests/test_doctor_cli.py \ + harness/tests/test_cli_config_environment.py harness/tests/test_memory_metadata.py \ + harness/tests/test_qdrant_cli_commands.py tools/tht/internal/setup/tls.go \ + tools/tht/internal/setup/tls_test.go tools/tht/internal/setup/run.go \ + tools/tht/internal/setup/run_test.go tools/tht/internal/doctor +git commit -m "feat(cli): harden diagnostics TLS and memory administration" +``` + +--- + +## Task 13: Rewrite and Verify the Pi Management Frontend Guidance + +**Files:** + +- Modify: `frontend/src/shell/PiManagement.tsx` +- Modify: `frontend/src/shell/PiManagement.test.tsx` +- Modify: `frontend/src/api/pi-management.ts` +- Modify: `frontend/src/api/pi-management.test.ts` +- Modify if required by the verified behavior: `backend/src/pi/management.ts` +- Modify if required by the verified behavior: `backend/src/routes/pi-management.ts` +- Modify if required by the verified behavior: `backend/test/pi-management.test.ts` +- Modify if required by the verified behavior: `backend/test/routes-pi-management.test.ts` + +### Information architecture and copy contract + +- The whole section starts collapsed. After the operator expands it, the section title remains “Update Pi configuration and relaunch its container.” +- Its introduction starts exactly with “Using the host terminal”. It must not say “not this browser page”. +- Linux, macOS, and Windows are three separate disclosure tabs/panels. All are closed on initial render; opening one does not require another to remain open. +- The containing window is shorter than the current one and has an always-available vertical scrollbar. Mouse wheel, trackpad, keyboard, and touch scrolling must not be trapped. +- Guidance covers only Pi managed by ThothII Docker Compose. Remove every native/local Pi branch. +- Explain in plain language that `deploy` is a directory in the root of the ThothII project, at the same level as `compose.yaml`. +- Explain that `deploy/pi/models.json` and `deploy/pi/settings.json` are host files mounted read-only into `core`: edit them from the host project/worktree, never from inside the container. +- Break every explanation longer than two rendered lines into short paragraphs, numbered steps, bullets, field tables, or command blocks. +- Explain configuration fields before naming them: + - `baseUrl`: the provider endpoint used by Pi. + - `api`: the Pi adapter/protocol expected by that provider. + - `models`: the provider's available model objects and identifiers. + - `enabledModels`: every `provider/model` pair that Pi may select. + - credential/auth file settings: paths to host-side files containing provider credentials; never paste a secret into the page, command, Compose file, or tracked JSON. +- Show direct commands only: `tht pi configure`, `tht pi update`, `tht pi status`, `tht pi doctor`, `tht pi test`, `tht pi logs`, and `tht pi rollback`. Do not show `thothctl`, `~/bin`, `./bin`, `./tht`, shell wrappers, Go builds, or mandatory `--installation`. +- State that commands work from a repository root or Git worktree root once `tht` has been installed. Explain the optional `--installation PATH` only as an ambiguity/automation override. +- State that omitting `--version` from `tht pi update` installs the latest stable Pi release. Do not conflate the Pi package version with the provider/model selected by `tht pi configure`. +- Preserve and display GLM 5.3 when it is present in the live Pi configuration. + +### Platform-specific command blocks + +Each panel uses the native terminal syntax but the same operation sequence: + +```text +1. Open Terminal/PowerShell in the ThothII repository or worktree root. +2. Edit deploy/pi/models.json and deploy/pi/settings.json on the host. +3. Run tht pi configure --provider --model --thinking . +4. Run tht pi update (or add --version X.Y.Z for an explicit Pi version). +5. Use tht pi restart when only configuration changed and no Pi package update is needed. +6. Run tht pi status, tht pi doctor, and tht pi test. +7. If needed, inspect tht pi logs or run tht pi rollback. +``` + +macOS and Windows explicitly require Docker Desktop to be running. Linux requires a running Docker Engine and permission to use Docker. PowerShell examples use PowerShell line continuation only when genuinely necessary; prefer one executable command per line. + +### “Show sanitized logs” behavior + +The button must request recent core/Pi diagnostic logs, display timestamps and severity where available, and redact credentials, authorization headers, API keys, tokens, cookies, connection strings, and secret file contents. It is diagnostic only: it must not change Pi or start/restart containers. Empty, unavailable, loading, success, and failure states must all be visible and understandable. If the current backend already satisfies this contract, change only tests/copy; otherwise make the smallest backend correction needed. + +### Steps + +- [ ] Extend component tests to require all platform panels closed initially, independent open/close state, a bounded scrollable region, exact introductory wording, structured copy, Docker-only guidance, root/worktree commands, field explanations, latest-Pi semantics, and absence of every obsolete command/path. +- [ ] Add accessibility tests for disclosure names, `aria-expanded`, focus order, keyboard activation, and scroll-region labeling. +- [ ] Extend API/backend tests for sanitized-log redaction and all UI states. Include adversarial fake logs containing every secret class listed above. +- [ ] Run focused tests and confirm current copy/layout fail the new contract. + +```bash +cd frontend +npx vitest run src/shell/PiManagement.test.tsx src/api/pi-management.test.ts +cd ../backend +npx vitest run test/pi-management.test.ts test/routes-pi-management.test.ts +``` + +- [ ] Refactor the long instruction blob into data-driven platform sections and small semantic components. Keep state local to Pi Management unless there is an existing shared disclosure component. +- [ ] Apply a bounded height plus `overflow-y: auto`/`scroll` to the actual element containing the full section; remove ancestor wheel/overflow rules that block movement. +- [ ] Implement or verify sanitized-log behavior end to end without exposing raw secrets to the browser. +- [ ] Rerun focused tests, type checks, builds, and complete frontend/backend suites. + +```bash +cd frontend +npx vitest run +npx tsc -b +npm run build +cd ../backend +npx vitest run +npx tsc --noEmit -p . +npm run build +``` + +- [ ] Commit the Pi Management UX slice without overwriting `deploy/pi/models.json` or `deploy/pi/settings.json`. + +```bash +git add frontend/src/shell/PiManagement.tsx frontend/src/shell/PiManagement.test.tsx \ + frontend/src/api/pi-management.ts frontend/src/api/pi-management.test.ts \ + backend/src/pi/management.ts backend/src/routes/pi-management.ts \ + backend/test/pi-management.test.ts backend/test/routes-pi-management.test.ts +git commit -m "feat(frontend): simplify Pi management guidance" +``` + +--- + +## Task 14: Replace Active Installation, CLI, and Pi Documentation + +**Files:** + +- Rename: `docs/contracts/thothctl-pi.md` → `docs/contracts/tht-pi.md` +- Modify: `README.md` +- Modify: `PROJECT_STATE.md` +- Modify: `AGENTS.md` +- Modify: `docs/architecture/overview.md` +- Modify: `docs/guida-utente.md` +- Modify: `docs/install/local.md` +- Modify: `docs/install/server.md` +- Modify: `docs/install/pi-management.md` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `docs/install/psd-workspace-setup.md` +- Modify: `docs/install/windows-line-endings.md` +- Modify: `docs/contracts/tht-dwh.md` +- Modify: `docs/contracts/workspace-preprocessing-cli.md` +- Modify: `docs/testing/p2-p6-manual-verification.md` +- Create: `scripts/test-tht-command-docs.sh` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/test-verify-workspace-install-docs.sh` + +### Required onboarding story + +After cloning ThothII, the normal path is exactly: + +```bash +# macOS or Linux, from the repository/worktree root +bash scripts/install-tht.sh +tht setup +``` + +```powershell +# Windows PowerShell, from the repository/worktree root +powershell -ExecutionPolicy Bypass -File scripts/install-tht.ps1 +tht setup +``` + +The documentation must say that `tht setup` validates Docker, creates local configuration, builds images, starts containers, waits for health, and runs diagnostics. It must present `--configure-only` as the explicit way to stop before build/start. `scripts/run-stack.sh` may remain documented as an advanced contributor shortcut, not the primary user onboarding path. + +### Steps + +- [ ] Create a documentation contract test that scans active docs and scripts for forbidden user instructions: `thothctl`, `~/bin`, `./bin/tht`, `./tht`, `tht.sh`, `go build`, a required `--installation`, native Pi setup, or editing files inside `core`. +- [ ] Exclude historical `docs/superpowers/specs/`, `docs/superpowers/plans/`, `docs/plans/`, and `docs/reports/` from stale-name failure; history remains immutable context. Active docs and code examples are not excluded. +- [ ] Add positive assertions for both installer commands, `tht setup`, `setup --configure-only`, `start --build`, full `update`, `backup`, `restore`, Pi latest-version behavior, worktree discovery, and the three Pi platforms. +- [ ] Run documentation tests and confirm they fail against current active guidance. + +```bash +bash scripts/test-tht-command-docs.sh +bash scripts/test-verify-workspace-install-docs.sh +``` + +- [ ] Rewrite active documents around one lifecycle: clone → install `tht` → `tht setup` → `tht doctor` → normal operation → `tht update`/`tht pi update` → `tht backup`/`tht restore`. +- [ ] Document the host/container command-name boundary once: users invoke the installed native `tht`; the backend and Pi gate invoke the Python `tht` inside `core`. Do not expose a second binary name. +- [ ] Document root/worktree discovery and optional `--installation` precedence with examples of ambiguity and automation, not as boilerplate on every command. +- [ ] Update the Pi contract and management guide with Docker-only host-file editing, declarative mounts, credential-file meaning, latest stable Pi default, rollback, and sanitized logs. +- [ ] Update command reference material from `approved_cli_surface.json`, explicitly omitting the 14 erased commands and marking enhanced administrative commands as advanced where applicable. +- [ ] Rerun documentation contracts and link checks. + +```bash +bash scripts/test-tht-command-docs.sh +bash scripts/test-verify-workspace-install-docs.sh +rg -n '\]\([^)]*\.md(#[^)]*)?\)' README.md PROJECT_STATE.md AGENTS.md docs/install docs/contracts docs/architecture docs/testing +``` + +- [ ] Commit active documentation and contracts. + +```bash +git add README.md PROJECT_STATE.md AGENTS.md docs/architecture docs/guida-utente.md \ + docs/install docs/contracts docs/testing scripts/test-tht-command-docs.sh \ + scripts/verify-workspace-install-docs.sh scripts/test-verify-workspace-install-docs.sh +git commit -m "docs: define the unified tht product workflow" +``` + +--- + +## Task 15: Run Cross-Platform Gates, Install on This Mac, and Update the Live Stack + +**Files:** + +- Create: `docs/reports/2026-08-15-unified-tht-cli-acceptance.md` +- Modify only if a gate finds a defect: files owned by Tasks 1–14, with a new failing regression test first + +### Automated acceptance matrix + +- Go: all native CLI unit, contract, transaction, race, and cross-platform compile tests. +- Python: Ruff and full non-L2 pytest; run opt-in L2 only when its external GLM/DWH prerequisites are available and record the result separately. +- Backend: Vitest, TypeScript no-emit typecheck, production build. +- Frontend: Vitest, TypeScript build, production build, Playwright. +- Deployment: Compose config for base+local, server, GPU, HTTPS/SSH workspace, preprocessing, and session-server variants. +- Installer: macOS/Linux shell tests and Windows PowerShell tests; cross-build native binaries. +- Documentation: active command/copy contracts and link/path checks. + +### Steps + +- [ ] Run the complete automated matrix from the repository/worktree root and save command, revision, result, and meaningful skips in the acceptance report. + +```bash +cd tools/tht && go test -race ./... && cd ../.. +cd harness && .venv/bin/ruff check . && .venv/bin/pytest -q && cd .. +cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build && cd .. +cd frontend && npx vitest run && npx tsc -b && npm run build && npm run e2e && cd .. +bash scripts/test-install-tht.sh +pwsh -NoProfile -File scripts/test-install-tht.ps1 +bash scripts/test-tht-command-docs.sh +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/unified-deployment-smoke.sh +``` + +- [ ] Cross-build and inspect all packaged binaries. Run Windows behavior tests in the existing Windows CI/VM path rather than claiming success from compilation alone. + +```bash +bash scripts/build-tht.sh --all +``` + +- [ ] Before touching the Mac installation, resolve the exact existing executables with `command -v`, `type -a`, file metadata, and hashes. Remove only the obsolete development `thothctl` binary or symlink that was positively identified; do not remove any directory or installation data. +- [ ] Install the newly tested Mac binary with `bash scripts/install-tht.sh`, start a fresh terminal lookup, and verify that `tht` resolves without a relative path while `thothctl` no longer resolves. + +```bash +command -v tht +type -a tht +tht version +tht help +``` + +- [ ] From `/Users/mp/projects/ThothII/.worktrees/p8-l2-live-session-smoke`, verify automatic worktree discovery and the optional installation override. Confirm no command searches for `/Users/mp/thothii-installation.yaml`. +- [ ] Inspect current sessions and container health. If no unsafe active work exists, update the live installation through the new transaction, using drain only as needed, then verify the stack. + +```bash +tht update --check-only +tht update --yes --drain +tht status +tht doctor +tht pi status +tht pi doctor +tht pi test +``` + +- [ ] Verify port 8080 in a real browser with Playwright: open Pi Management; require all three platform panels closed initially; open each independently; scroll the bounded panel using wheel and keyboard; verify structured Linux/macOS/Windows text; verify only direct `tht` commands; click “Show sanitized logs” and inspect loading/success/empty/error behavior; confirm GLM 5.3 remains available; require no console errors, failed API calls, or exposed secrets. +- [ ] Compare the live Pi configuration and provider/model list before and after deployment to prove the existing GLM 5.3 changes were preserved. +- [ ] Record exact versions, image digests, test totals, manual observations, live URL, rollback checkpoint, and any explicitly deferred L2/Windows gate in `docs/reports/2026-08-15-unified-tht-cli-acceptance.md`. +- [ ] Run `git status --short` and a final diff audit. Confirm no unrelated files, secrets, `.playwright-cli/`, or stale model fixtures were staged. +- [ ] Commit only the acceptance report and regression fixes, if any. + +```bash +git add docs/reports/2026-08-15-unified-tht-cli-acceptance.md +git commit -m "test: record unified tht acceptance" +``` + +--- + +## Approval Boundary + +Approval of this plan authorizes implementation of Tasks 1–15 in order, including: + +- replacing `thothctl` with the single installed command `tht` without compatibility aliases; +- installing the finished CLI on this Mac after automated gates pass; +- removing only the positively identified obsolete Mac development executable; +- updating/recreating the live Docker Compose services and verifying the GUI on port 8080; +- removing the 14 approved workflow CLI commands and implementing the 8 approved enhancements; +- changing active project documentation and Pi Management guidance as specified; +- creating backup/restore checkpoints needed to make updates recoverable. + +Approval does **not** authorize deleting user data, secrets, workspaces, sessions, unrelated worktree changes, or historical design/audit documents. It does not authorize overwriting the current GLM 5.3 configuration. Any newly discovered decision that materially changes this architecture, command surface, data-safety policy, or live-deployment scope must return to the user for approval before implementation continues. + +Implementation should stop for review after these four milestones: + +1. Tasks 1–5: one installable command and complete setup. +2. Tasks 6–10: diagnostics, Pi/product updates, backup, restore, and rollback. +3. Tasks 11–14: approved command-surface changes, frontend, and documentation. +4. Task 15: Mac installation and live acceptance on port 8080. diff --git a/docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md b/docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md new file mode 100644 index 00000000..1ff3dd47 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md @@ -0,0 +1,44 @@ +# Simplified `thothctl` installation selection and Pi update design + +## Decision + +Keep every existing `thothctl pi` subcommand. Make the installation descriptor optional on the command line and resolve it automatically when the operator runs from the project tree. Keep `--installation ` as an explicit override for non-standard locations or multiple installations. + +The normal Pi update becomes: + +```sh +thothctl pi update +``` + +When `--version` is omitted, `thothctl` reads the single default `ARG PI_VERSION=` from `docker/core.Dockerfile` in the selected installation's project directory and uses that pinned version with the existing transactional build/update path. An explicit `--version ` remains supported. The command does not fetch an arbitrary npm “latest”; the repository pin, lockfile, image labels, and executable must remain consistent. + +## Installation resolution + +Resolution order is: + +1. an explicit `--installation /thothii-installation.yaml`; +2. `THOTHII_INSTALLATION`, when set to an absolute descriptor path; +3. automatic discovery from the current working directory and its parents. + +Automatic discovery examines only the exact descriptor at each directory level and the immediate `deploy/*/thothii-installation.yaml` locations. It never recursively scans `.artifacts`, home directories, or unrelated descendants. A candidate must be a regular file and must pass `config.Load`. One valid candidate is selected. No candidates or multiple valid candidates produce an actionable error that names the expected locations and explains how to use `--installation`. + +The resolver is shared by all existing top-level commands, not only `pi`, so `thothctl status`, `start`, `stop`, `doctor`, `workspace`, and the Pi commands have the same invocation rules. Existing explicit invocations remain valid. + +## Safety and compatibility + +- No existing Pi subcommand is removed or renamed. +- Existing advanced `pi update --source ... --image ... --yes --drain` syntax remains accepted for compatibility. +- The short update path selects build mode and preserves the existing lifecycle lock, maintenance gate, session handling, candidate verification, image selector promotion, and rollback/recovery behavior. +- The default update uses the current checkout's declared Pi pin; changing to a newer Pi release still requires updating the repository pin and lockfile in the normal source-update workflow. +- Errors and automatic-discovery diagnostics never expose secret file contents. + +## User-facing examples + +```sh +thothctl pi update +thothctl pi update --version 0.81.0 +thothctl pi status +thothctl pi restart --yes --drain +thothctl --installation ~/operator/thothii-installation.yaml pi update +``` + diff --git a/docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md b/docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md new file mode 100644 index 00000000..aee2b137 --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md @@ -0,0 +1,235 @@ +# Unified `tht` CLI and Product Setup Design + +Date: 2026-08-15 + +## Objective + +Turn the repository into a product that can be cloned, bootstrapped once, and then operated with a +single normal system command named `tht`. The operator must not build Go manually, create a `bin` +directory, remember an installation descriptor path, or use raw Docker commands for normal +installation, lifecycle, Pi updates, backup, or restore. + +## Confirmed decisions + +1. The only product command name is `tht`. +2. `thothctl` is removed completely. There is no compatibility alias, wrapper, deprecation period, + or second installed command. +3. The host operator implementation remains a native Go binary so macOS, Linux, and Windows hosts + do not require Go, Python, or a virtualenv. +4. The existing Python workflow CLI remains inside `core` under the same command name `tht`. + Backend and Pi continue to use it there. It is not installed on the host and is not presented in + operator documentation. No `tht-runtime` command or namespace is introduced. +5. The command audit is binding: 55 commands are `MAINTAIN`, 8 are `ENHANCE`, and 14 are `ERASE`. + The detailed decision matrix is in + `docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md`. +6. Workflow commands called by Pi, the gate, the backend, workspace maintenance, or the session + migrator are machine contracts. Their semantics, pristine JSON output, stdin behavior, and exit + codes are not changed merely to simplify the operator help. +7. The public operator help stays small. Internal workflow commands do not appear in host help. +8. `--installation` remains available as an optional override. Normal commands discover the + installation from the current project root or worktree. +9. `backup` and `restore` are introduced as first-class product commands. +10. The final implementation is installed on this Mac and used to rebuild/restart the live stack on + port 8080. + +## Command boundary + +### Host operator CLI + +The native host command exposes: + +```text +tht setup +tht version +tht start [--build] +tht stop +tht status +tht doctor [--json] +tht logs +tht update [--check-only] [--yes] [--drain] +tht backup [--output PATH] [--include-secrets --yes] [--drain] +tht restore ARCHIVE --yes [--drain] +tht sessions migrate --yes +tht remove [--yes ID...] +tht pi ... +tht workspace ... +``` + +`tht pi` preserves `status`, `doctor`, `test`/`check`, `configure`, `restart`, `update`, +`rollback`, `maintenance`, and `logs`. `tht workspace` preserves every currently implemented +workspace operation, including the two vector operations missing from the current help. + +### Container workflow CLI + +The Python command tree remains the deterministic protocol used by Pi and the backend. The host +installation does not expose these commands as operator shortcuts. This prevents a human operator +from bypassing reviewer gates while avoiding a risky rewrite of the live workflow. + +The `MAINTAIN`, `ENHANCE`, and `ERASE` decisions apply exactly as recorded in the command audit. +`MAINTAIN` commands retain their current path. `ENHANCE` commands receive the approved safety or +diagnostic improvements. `ERASE` commands disappear from Typer registration and active docs; domain +functions still used by canonical pipelines remain internal libraries. + +## Bootstrap and PATH installation + +A freshly cloned repository necessarily needs one bootstrap action before `tht` exists: + +```bash +bash scripts/install-tht.sh +``` + +Windows uses: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/install-tht.ps1 +``` + +The scripts require Docker, build the correct native binary using the repository-pinned Docker +builder, verify it, and install it atomically: + +- macOS and Linux: `/usr/local/bin/tht`, requesting elevation only for the final atomic install; +- Windows: `%LOCALAPPDATA%\ThothII\bin\tht.exe`, adding that directory to the user PATH when needed. + +The user never selects a platform binary, runs Go, creates a `bin` directory, or invokes a +project-relative executable. Re-running the installer upgrades the installed command idempotently. + +## Project and worktree discovery + +Every host command starts from the current working directory and walks parents until it finds the +ThothII root contract (`compose.yaml`, `deploy/`, and the repository marker). A Git worktree root is +treated exactly like the main checkout. + +Installation resolution order is: + +1. explicit `--installation PATH`; +2. `THOTHII_INSTALLATION`; +3. one valid `thothii-installation.yaml` in the current directory or immediate `deploy/*`; +4. the same search while walking parent directories. + +Zero candidates produce a setup-oriented error. Multiple candidates produce a bounded list and +require `--installation`; no arbitrary recursive search is allowed. + +## `tht setup` + +`tht setup` is idempotent and defaults to the local profile on macOS, Windows, and workstation +Linux. `--profile server` selects server behavior. Generated installation files live under +`deploy//`, where `deploy` is explicitly documented as a directory in the project +or worktree root. + +The setup flow: + +1. verifies root/worktree identity, Docker, Compose, supported architecture, and line endings; +2. creates or validates the installation descriptor and non-secret environment files; +3. asks plain-language questions and stores secret file paths, never secret values in the + descriptor; +4. creates protected secret-file templates only after explicit confirmation and never overwrites an + existing file; +5. renders and validates Compose configuration; +6. builds the ThothII images from the current checkout; +7. starts the stack with `docker compose up --detach --remove-orphans`; +8. waits for bounded health checks; +9. runs installation diagnostics and Pi diagnostics; +10. prints the URL and the exact descriptor selected. + +`tht setup --configure-only` stops after validated configuration. `tht setup` never silently +replaces a descriptor, environment file, secret file, or generated state belonging to another +installation. + +## Lifecycle and product update + +- `tht start` starts the selected installation without rebuilding. +- `tht start --build` builds current-checkout images before startup. +- `tht stop` stops the installation while preserving state. +- `tht update --check-only` retains its current non-mutating validation behavior. +- `tht update` becomes the complete product update: lifecycle lock, active-session check/drain, + backup checkpoint, image build from the current checkout, controlled recreation, health checks, + diagnostics, and rollback to the recorded images when verification fails. + +Product update and Pi update remain separate. `tht update` updates ThothII. `tht pi update` updates +only Pi in `core`. + +## Pi management + +`tht pi update [--version VERSION]` makes the version optional. Without `--version`, it queries the +latest stable version of the pinned Pi package from the authoritative package registry. A lookup +failure stops before mutation and tells the operator to retry or supply `--version`; it never +silently substitutes an older pin. + +The command builds or pulls a candidate, verifies the Pi executable, `PI_VERSION`, and image label, +recreates only `core`, checks health, runs the smoke test, preserves volumes, and rolls back on +failure. Interactive terminals receive one clear confirmation; non-interactive execution requires +`--yes`. Model/provider selection remains the responsibility of `tht pi configure` and is not a +required argument to Pi update. + +The project release pin in `docker/core.Dockerfile` remains the clean-build default. Installation +update state records the selected newer image/version so ordinary restart does not revert it. + +## Backup and restore + +`tht backup` creates a versioned manifest and checksummed archive in +`~/.thothii/backups//` unless `--output` is supplied. It acquires the lifecycle +lock, refuses active work unless `--drain` is accepted, obtains a consistent stopped snapshot, and +restarts/verifies a previously running installation. + +The backup includes: + +- installation descriptor, non-secret environment/configuration, generated overrides, and source + revision metadata; +- installation-owned settings, Pi state, workspace registry, workspace secrets volume, sessions, + Qdrant data, and embedding-model volume; +- server bind roots returned by the installation preservation contract; +- a manifest of external secret-file paths and digests. + +Secret-file contents are excluded by default. `--include-secrets --yes` includes them and marks the +archive sensitive; the file is created with owner-only permissions. A backup without secrets is +restorable only when all referenced secret files still exist and match preflight requirements. + +`tht restore ARCHIVE --yes` validates schema version, checksums, installation identity, target +ownership, secret prerequisites, disk space, and stopped/quiescent state before mutation. It creates +a rollback checkpoint, restores only manifest-listed paths/volumes, starts the stack when it was +previously running, and runs health, `doctor`, `pi doctor`, and workspace inspection. Failure keeps +the target in a recoverable stopped state and prints the checkpoint path. + +## Pi Management frontend + +The frontend uses only Docker-based instructions and only the command `tht`. The section: + +- begins fully collapsed; +- uses separate Linux, macOS, and Windows environment panels, with none open initially; +- is shorter than the current panel and has a working vertical scrollbar; +- begins with the exact concise wording `Using the host terminal`; +- explains that `deploy` is a directory in the project/worktree root beside `compose.yaml`; +- explains that `deploy/pi/models.json` and `deploy/pi/settings.json` are host files mounted + read-only into `core`, so the operator edits the host files, not files inside the container; +- explains credential-file concepts in plain language rather than presenting environment-variable + names without context; +- shows direct commands such as `tht pi configure`, `tht pi restart`, `tht pi update`, + `tht pi status`, `tht pi doctor`, and `tht pi test`; +- contains no Go build, `~/bin`, `./bin`, `./tht`, `thothctl`, or mandatory descriptor path. + +The sanitized-log control must either display the bounded, sanitized `core` log response or fail +with a visible error. The live model selector must reflect the mounted Pi configuration, including +the existing GLM 5.3 change after `core` is recreated. + +## Documentation + +Active user, installation, architecture, CLI-contract, testing, and README documentation is +rewritten around the bootstrap-plus-setup flow and the direct `tht` command. Historical +`docs/superpowers` plans/specs remain historical records; the new spec supersedes them. + +Documentation examples run from the project/worktree root, refer to the home directory as `~`, and +do not teach manual Go builds or manually constructed installation paths for normal use. Advanced +sections may document optional `--installation` and non-interactive flags. + +## Safety and acceptance + +- Existing user changes to `deploy/pi/models.json` and `deploy/pi/settings.json` are preserved. +- Unrelated dirty-worktree files are not overwritten or committed accidentally. +- JSON contracts remain pristine and secrets are sanitized from stdout, stderr, logs, archives, and + failure messages. +- Tests cover macOS/Linux shell installation, Windows PowerShell installation, root/worktree + discovery, setup idempotency, lifecycle rollback, Pi latest-version lookup, command audit, + backup/restore, frontend layout/copy, and active-document command examples. +- Final acceptance installs `tht` on this Mac, verifies `command -v tht`, confirms `thothctl` is + absent, updates the live stack, checks healthy services, opens port 8080, verifies Pi Management, + and confirms GLM 5.3 is selectable. diff --git a/frontend/src/shell/PiManagement.test.tsx b/frontend/src/shell/PiManagement.test.tsx index f633df11..17700591 100644 --- a/frontend/src/shell/PiManagement.test.tsx +++ b/frontend/src/shell/PiManagement.test.tsx @@ -238,25 +238,30 @@ test("shows a seven-step host-terminal workflow in scrollable platform tabs", as "Open the project root", "Edit the provider catalog", "Enable the model", - "Set the provider credential", + "Check the provider credential", "Reload Pi configuration", "Update the Pi version", "Recover a failed update", ]); - expect(linux).toHaveTextContent("The deploy directory is in the ThothII project root, beside compose.yaml"); + expect(linux).toHaveTextContent("The deploy directory is beside compose.yaml"); + expect(linux).toHaveTextContent("All commands below start in this project root"); + expect(linux).toHaveTextContent("mkdir -p bin"); + expect(linux).toHaveTextContent("go -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl"); expect(linux).toHaveTextContent("deploy/pi/models.json"); expect(linux).toHaveTextContent("deploy/pi/settings.json"); expect(linux).toHaveTextContent("baseUrl is the provider API endpoint"); expect(linux).toHaveTextContent("enabledModels uses provider/model identifiers"); - expect(linux).toHaveTextContent("PI_AUTH_FILE is a setting in the installation environment file"); - expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi restart --yes --drain"); - expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source build --yes --drain"); - expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source pull --image @sha256: --yes --drain"); - expect(linux).toHaveTextContent(" is a placeholder. Replace it with the Pi release/version you want to install."); - expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance status"); - expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi logs"); - expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi rollback --yes"); - expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance recover --yes"); + expect(linux).toHaveTextContent("Pi reads the provider API key from a protected file on the host"); + expect(linux).toHaveTextContent("Do not put the key in models.json or settings.json"); + expect(linux).toHaveTextContent("./bin/thothctl pi restart --yes --drain"); + expect(linux).toHaveTextContent("./bin/thothctl pi update"); + expect(linux).toHaveTextContent("./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain"); + expect(linux).toHaveTextContent("The command installs the Pi version pinned in docker/core.Dockerfile"); + expect(linux).toHaveTextContent("./bin/thothctl pi maintenance status"); + expect(linux).toHaveTextContent("./bin/thothctl pi logs"); + expect(linux).toHaveTextContent("./bin/thothctl pi rollback --yes"); + expect(linux).toHaveTextContent("./bin/thothctl pi maintenance recover --yes"); + expect(linux).not.toHaveTextContent("~/bin/"); expect(linux).not.toHaveTextContent("~/.pi/agent/"); await user.click(macosTab); @@ -266,22 +271,24 @@ test("shows a seven-step host-terminal workflow in scrollable platform tabs", as "Open the project root", "Edit the provider catalog", "Enable the model", - "Set the provider credential", + "Check the provider credential", "Reload Pi configuration", "Update the Pi version", "Recover a failed update", ]); - expect(macos).toHaveTextContent("The deploy directory is in the ThothII project root, beside compose.yaml"); + expect(macos).toHaveTextContent("The deploy directory is beside compose.yaml"); + expect(macos).toHaveTextContent("All commands below start in this project root"); + expect(macos).toHaveTextContent("go -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl"); expect(macos).toHaveTextContent("deploy/pi/models.json"); expect(macos).toHaveTextContent("deploy/pi/settings.json"); - expect(macos).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi restart --yes --drain"); - expect(macos).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source build --yes --drain"); - expect(macos).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source pull --image @sha256: --yes --drain"); - expect(macos).toHaveTextContent(" is a placeholder. Replace it with the Pi release/version you want to install."); - expect(macos).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance status"); - expect(macos).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi logs"); - expect(macos).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi rollback --yes"); - expect(macos).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance recover --yes"); + expect(macos).toHaveTextContent("./bin/thothctl pi restart --yes --drain"); + expect(macos).toHaveTextContent("./bin/thothctl pi update"); + expect(macos).toHaveTextContent("./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain"); + expect(macos).toHaveTextContent("The command installs the Pi version pinned in docker/core.Dockerfile"); + expect(macos).toHaveTextContent("./bin/thothctl pi maintenance status"); + expect(macos).toHaveTextContent("./bin/thothctl pi logs"); + expect(macos).toHaveTextContent("./bin/thothctl pi rollback --yes"); + expect(macos).toHaveTextContent("./bin/thothctl pi maintenance recover --yes"); await user.click(windowsTab); const windows = screen.getByRole("tabpanel", { name: "Windows" }); @@ -290,22 +297,26 @@ test("shows a seven-step host-terminal workflow in scrollable platform tabs", as "Open the project root", "Edit the provider catalog", "Enable the model", - "Set the provider credential", + "Check the provider credential", "Reload Pi configuration", "Update the Pi version", "Recover a failed update", ]); - expect(windows).toHaveTextContent("The deploy directory is in the ThothII project root, beside compose.yaml"); + expect(windows).toHaveTextContent("The deploy directory is beside compose.yaml"); + expect(windows).toHaveTextContent("All commands below start in this project root"); expect(windows).toHaveTextContent("deploy\\pi\\models.json"); expect(windows).toHaveTextContent("deploy\\pi\\settings.json"); - expect(windows).toHaveTextContent('& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi restart --yes --drain'); - expect(windows).toHaveTextContent('& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi update --version --source build --yes --drain'); - expect(windows).toHaveTextContent('& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi update --version --source pull --image @sha256: --yes --drain'); - expect(windows).toHaveTextContent(" is a placeholder. Replace it with the Pi release/version you want to install."); - expect(windows).toHaveTextContent('& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi maintenance status'); - expect(windows).toHaveTextContent('& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi logs'); - expect(windows).toHaveTextContent('& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi rollback --yes'); - expect(windows).toHaveTextContent('& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi maintenance recover --yes'); + expect(windows).toHaveTextContent("New-Item -ItemType Directory -Force bin"); + expect(windows).toHaveTextContent("go -C tools/thothctl build -o ../../bin/thothctl.exe ./cmd/thothctl"); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi restart --yes --drain'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi update'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi update --version --source pull --image @sha256: --yes --drain'); + expect(windows).toHaveTextContent("The command installs the Pi version pinned in docker/core.Dockerfile"); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi maintenance status'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi logs'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi rollback --yes'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi maintenance recover --yes'); + expect(windows).not.toHaveTextContent("~\\bin\\"); }); test("keeps the required PI_AUTH_FILE guidance in one static text node", async () => { @@ -314,13 +325,13 @@ test("keeps the required PI_AUTH_FILE guidance in one static text node", async ( const tablist = await screen.findByRole("tablist", { name: "Pi host platform" }); await user.click(within(tablist).getByRole("tab", { name: "Linux" })); - const credentialStep = screen.getByRole("heading", { name: "Set the provider credential" }).closest("li"); + const credentialStep = screen.getByRole("heading", { name: "Check the provider credential" }).closest("li"); const guidance = credentialStep?.querySelector("p"); expect( Array.from(guidance?.childNodes ?? []).some( (node) => node.nodeType === Node.TEXT_NODE - && node.textContent?.includes("PI_AUTH_FILE is a setting in the installation environment file"), + && node.textContent?.includes("Pi reads the provider API key from a protected file on the host"), ), ).toBe(true); }); diff --git a/frontend/src/shell/PiManagement.tsx b/frontend/src/shell/PiManagement.tsx index 98226c4e..19baf848 100644 --- a/frontend/src/shell/PiManagement.tsx +++ b/frontend/src/shell/PiManagement.tsx @@ -108,6 +108,7 @@ type PiPlatformDetails = { settingsPath: string; terminal: string; credentialProtection: string; + buildCommand: string; restartCommand: string; updateCommand: string; pullCommand: string; @@ -123,10 +124,11 @@ const piPlatforms: Array<{ id: PiPlatform; label: string; details: PiPlatformDet settingsPath: "deploy/pi/settings.json", terminal: "a terminal", credentialProtection: "a protected host file with mode 0600", - restartCommand: "~/bin/thothctl --installation ~/thothii-installation.yaml pi restart --yes --drain", - updateCommand: "~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source build --yes --drain", - pullCommand: "~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source pull --image @sha256: --yes --drain", - recoveryCommands: "~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance status\n~/bin/thothctl --installation ~/thothii-installation.yaml pi logs\n~/bin/thothctl --installation ~/thothii-installation.yaml pi rollback --yes\n~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance recover --yes", + buildCommand: "mkdir -p bin\ngo -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl", + restartCommand: "./bin/thothctl pi restart --yes --drain", + updateCommand: "./bin/thothctl pi update", + pullCommand: "./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain", + recoveryCommands: "./bin/thothctl pi maintenance status\n./bin/thothctl pi logs\n./bin/thothctl pi rollback --yes\n./bin/thothctl pi maintenance recover --yes", }, }, { @@ -137,10 +139,11 @@ const piPlatforms: Array<{ id: PiPlatform; label: string; details: PiPlatformDet settingsPath: "deploy/pi/settings.json", terminal: "Terminal", credentialProtection: "a protected host file with mode 0600", - restartCommand: "~/bin/thothctl --installation ~/thothii-installation.yaml pi restart --yes --drain", - updateCommand: "~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source build --yes --drain", - pullCommand: "~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source pull --image @sha256: --yes --drain", - recoveryCommands: "~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance status\n~/bin/thothctl --installation ~/thothii-installation.yaml pi logs\n~/bin/thothctl --installation ~/thothii-installation.yaml pi rollback --yes\n~/bin/thothctl --installation ~/thothii-installation.yaml pi maintenance recover --yes", + buildCommand: "mkdir -p bin\ngo -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl", + restartCommand: "./bin/thothctl pi restart --yes --drain", + updateCommand: "./bin/thothctl pi update", + pullCommand: "./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain", + recoveryCommands: "./bin/thothctl pi maintenance status\n./bin/thothctl pi logs\n./bin/thothctl pi rollback --yes\n./bin/thothctl pi maintenance recover --yes", }, }, { @@ -151,10 +154,11 @@ const piPlatforms: Array<{ id: PiPlatform; label: string; details: PiPlatformDet settingsPath: "deploy\\pi\\settings.json", terminal: "PowerShell", credentialProtection: "a protected host file with a user-only ACL", - restartCommand: '& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi restart --yes --drain', - updateCommand: '& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi update --version --source build --yes --drain', - pullCommand: '& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi update --version --source pull --image @sha256: --yes --drain', - recoveryCommands: '& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi maintenance status\n& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi logs\n& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi rollback --yes\n& (Resolve-Path "~\\bin\\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\\thothii-installation.yaml") pi maintenance recover --yes', + buildCommand: "New-Item -ItemType Directory -Force bin | Out-Null\ngo -C tools/thothctl build -o ../../bin/thothctl.exe ./cmd/thothctl", + restartCommand: ".\\bin\\thothctl.exe pi restart --yes --drain", + updateCommand: ".\\bin\\thothctl.exe pi update", + pullCommand: ".\\bin\\thothctl.exe pi update --version --source pull --image @sha256: --yes --drain", + recoveryCommands: ".\\bin\\thothctl.exe pi maintenance status\n.\\bin\\thothctl.exe pi logs\n.\\bin\\thothctl.exe pi rollback --yes\n.\\bin\\thothctl.exe pi maintenance recover --yes", }, }, ]; @@ -167,7 +171,9 @@ function PiInstructionSteps({ details }: { details: PiPlatformDetails }) { return

  1. Open the project root

    -

    Open {details.terminal} in the ThothII project root. The deploy directory is in the ThothII project root, beside compose.yaml.

    +

    Using {details.terminal}, open the current ThothII checkout or worktree root. All commands below start in this project root. The deploy directory is beside compose.yaml. If this checkout has no bin directory, create it and build the local CLI once:

    + {details.buildCommand} +

    The commands below use the executable built in this checkout, so they cannot accidentally target another worktree.

  2. Edit the provider catalog

    @@ -188,8 +194,8 @@ function PiInstructionSteps({ details }: { details: PiPlatformDetails }) {
  3. -

    Set the provider credential

    -

    PI_AUTH_FILE is a setting in the installation environment file. It selects {details.credentialProtection}; Docker mounts the selected host credential file read-only for Pi.

    +

    Check the provider credential

    +

    Pi reads the provider API key from a protected file on the host. PI_AUTH_FILE tells this installation which file to use; Docker mounts it read-only into the core container. Do not put the key in models.json or settings.json. Keep {details.credentialProtection}.

  4. Reload Pi configuration

    @@ -198,8 +204,7 @@ function PiInstructionSteps({ details }: { details: PiPlatformDetails }) {

  5. Update the Pi version

    -

    Use a build update only when changing the bundled Pi version.

    -

    {" is a placeholder. Replace it with the Pi release/version you want to install."}

    +

    The command installs the Pi version pinned in docker/core.Dockerfile. Use --version <VERSION> only when you deliberately want another version.

    {details.updateCommand}

    Advanced: pull an immutable, digest-pinned image.

    {details.pullCommand} diff --git a/frontend/src/shell/WorkspaceManager.test.tsx b/frontend/src/shell/WorkspaceManager.test.tsx index 05942e35..96e821c5 100644 --- a/frontend/src/shell/WorkspaceManager.test.tsx +++ b/frontend/src/shell/WorkspaceManager.test.tsx @@ -114,7 +114,13 @@ test("level one explains the read-only Git sequence and the repository update bu const overview = screen.getByTestId("workspace-overview"); expect(within(overview).getByText(/Git server such as GitHub, GitLab, or Gitea/i)).toBeVisible(); expect(within(overview).getByText(/configured during ThothII installation/i)).toBeVisible(); - expect(within(overview).getAllByRole("listitem")[0]).toHaveTextContent(/choosing any local directory you prefer/i); + const repositoryStep = within(overview).getAllByRole("listitem")[0]; + expect(repositoryStep).toHaveTextContent(/create a workspace repository/i); + expect(repositoryStep).toHaveTextContent(/one directory for each workspace/i); + expect(repositoryStep).toHaveTextContent(/thoth-workspaces\.yaml/i); + expect(repositoryStep).toHaveTextContent(/database connection/i); + expect(repositoryStep).toHaveTextContent(/Evidence sources/i); + expect(repositoryStep).toHaveTextContent(/vector-database collection/i); expect(within(overview).getByRole("link", { name: /workspace authoring instructions on GitHub/i })).toHaveAttribute( "href", "https://github.com/mptyl/ThothII/blob/main/docs/install/local-workspace-registry.md#prepare-and-publish-a-workspace-source", @@ -126,7 +132,7 @@ test("level one explains the read-only Git sequence and the repository update bu await user.click(screen.getByRole("button", { name: "Update workspace repository" })); expect(await screen.findByText("Workspace repository updated and validated.")).toBeVisible(); - expect(screen.getByText(/create a local workspace/i)).toBeInTheDocument(); + expect(screen.getByText(/create a workspace repository/i)).toBeInTheDocument(); expect(screen.queryByText(/import|export|bundle/i)).not.toBeInTheDocument(); }); @@ -141,10 +147,61 @@ test("workspace-specific commands remain isolated until a workspace is selected" expect(screen.getByText(/reads this revision without modifying or publishing it/i)).toBeVisible(); expect(screen.getByText(/checks workspace.yaml and the required workspace directories/i)).toBeVisible(); expect(screen.getByText(/temporary decrypted credentials/i)).toBeVisible(); + const databaseField = screen.getByText("Database").parentElement; + expect(databaseField).not.toBeNull(); + expect(databaseField).toHaveTextContent("engine: postgres"); + expect(databaseField).toHaveTextContent("database: database"); + expect(databaseField).toHaveTextContent("schema: public"); expect(screen.getByRole("button", { name: "Validate workspace source" })).toBeVisible(); expect(screen.getByRole("button", { name: "Test workspace connections" })).toBeVisible(); }); +test("keeps validation and connection results inside their respective action cards", async () => { + const user = userEvent.setup(); + server.use( + http.post("/api/workspaces/validate", () => HttpResponse.json({ workspace, contract: {} })), + http.post("/api/workspaces/psd-clinical/test", () => HttpResponse.json({ + activatable: false, + diagnostics: [{ level: "error", code: "connector_unavailable", message: "Connector diagnostic failed." }], + })), + ); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + const validationCard = screen.getByTestId("workspace-validation-card"); + const connectionCard = screen.getByTestId("workspace-connection-card"); + await user.click(within(validationCard).getByRole("button", { name: "Validate workspace source" })); + const validationStatus = await within(validationCard).findByRole("status"); + expect(validationStatus).toHaveTextContent("Workspace source is valid."); + expect(validationStatus).toHaveClass("text-emerald-700"); + expect(within(connectionCard).queryByText("Workspace source is valid.")).not.toBeInTheDocument(); + + await user.click(within(connectionCard).getByRole("button", { name: "Test workspace connections" })); + expect(await within(connectionCard).findByRole("alert")).toHaveTextContent( + "connector_unavailable: Connector diagnostic failed.", + ); + expect(within(validationCard).queryByText("connector_unavailable: Connector diagnostic failed.")).not.toBeInTheDocument(); +}); + +test("renders binding_ok as a green connection success", async () => { + const user = userEvent.setup(); + server.use( + http.post("/api/workspaces/psd-clinical/test", () => HttpResponse.json({ + activatable: true, + diagnostics: [{ level: "info", code: "binding_ok", message: "Installation bindings and diagnostics succeeded." }], + })), + ); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + const connectionCard = screen.getByTestId("workspace-connection-card"); + await user.click(within(connectionCard).getByRole("button", { name: "Test workspace connections" })); + const connectionStatus = await within(connectionCard).findByRole("status"); + expect(connectionStatus).toHaveTextContent("binding_ok: Installation bindings and diagnostics succeeded."); + expect(connectionStatus).toHaveClass("text-emerald-700"); + expect(within(connectionCard).queryByRole("alert")).not.toBeInTheDocument(); +}); + test("secret fields are write-only, clear after blind save, and may be forgotten", async () => { const user = userEvent.setup(); let savedBody: unknown; diff --git a/frontend/src/shell/WorkspaceManager.tsx b/frontend/src/shell/WorkspaceManager.tsx index afc01b0f..fa455ad4 100644 --- a/frontend/src/shell/WorkspaceManager.tsx +++ b/frontend/src/shell/WorkspaceManager.tsx @@ -66,6 +66,10 @@ export function WorkspaceManager({ open, onClose }: { open: boolean; onClose: () const [secretValues, setSecretValues] = useState>({}); const [notice, setNotice] = useState(); const [diagnostics, setDiagnostics] = useState([]); + const [validationNotice, setValidationNotice] = useState(); + const [validationDiagnostics, setValidationDiagnostics] = useState([]); + const [connectionNotice, setConnectionNotice] = useState(); + const [connectionDiagnostics, setConnectionDiagnostics] = useState([]); const [busyAction, setBusyAction] = useState(); const statusQuery = useQuery({ @@ -97,6 +101,15 @@ export function WorkspaceManager({ open, onClose }: { open: boolean; onClose: () const clearMessages = () => { setNotice(undefined); setDiagnostics([]); + setValidationNotice(undefined); + setValidationDiagnostics([]); + setConnectionNotice(undefined); + setConnectionDiagnostics([]); + }; + + const clearGlobalMessages = () => { + setNotice(undefined); + setDiagnostics([]); }; const close = () => { @@ -139,12 +152,14 @@ export function WorkspaceManager({ open, onClose }: { open: boolean; onClose: () async function validateSource() { if (!detailQuery.data) return; setBusyAction("validate"); - clearMessages(); + clearGlobalMessages(); + setValidationNotice(undefined); + setValidationDiagnostics([]); try { await validateWorkspace(detailQuery.data.workspace); - setNotice("Workspace source is valid."); + setValidationNotice("Workspace source is valid."); } catch (error) { - setDiagnostics([publicError(error, "workspace_invalid: Workspace validation could not be completed")]); + setValidationDiagnostics([publicError(error, "workspace_invalid: Workspace validation could not be completed")]); } finally { setBusyAction(undefined); } @@ -153,17 +168,23 @@ export function WorkspaceManager({ open, onClose }: { open: boolean; onClose: () async function testConnections() { if (!selectedId) return; setBusyAction("test"); - clearMessages(); + clearGlobalMessages(); + setConnectionNotice(undefined); + setConnectionDiagnostics([]); try { const result = await testWorkspace(selectedId); - setDiagnostics(result.diagnostics.map(({ code, message }) => `${code}: ${message}`)); - if (result.diagnostics.length === 0) { - setNotice(result.activatable - ? "Workspace connections are valid." + const issues = result.diagnostics.filter(({ level }) => level !== "info"); + const informational = result.diagnostics.find(({ level }) => level === "info"); + setConnectionDiagnostics(issues.map(({ code, message }) => `${code}: ${message}`)); + if (issues.length === 0) { + setConnectionNotice(result.activatable + ? informational + ? `${informational.code}: ${informational.message}` + : "Workspace connections are valid." : "Workspace connection test completed."); } } catch (error) { - setDiagnostics([publicError(error, "connector_unavailable: Workspace connections could not be tested")]); + setConnectionDiagnostics([publicError(error, "connector_unavailable: Workspace connections could not be tested")]); } finally { setBusyAction(undefined); } @@ -293,7 +314,7 @@ export function WorkspaceManager({ open, onClose }: { open: boolean; onClose: ()

    How workspaces reach ThothII

      -
    1. Create a local workspace by choosing any local directory you prefer and setting up the workspace there. It must contain workspace.yaml and every required subdirectory, including any versioned Evidence files.
      2. +
    2. Create a workspace repository in any local directory you choose. Add one directory for each workspace you want ThothII to manage. At the repository root, thoth-workspaces.yaml lists those workspaces; each workspace directory contains its own workspace.yaml, which declares the database connection, the Evidence sources, and the ThothII vector-database collection used during the process.
    3. Publish that source by committing and pushing it to a repository hosted by a Git server such as GitHub, GitLab, or Gitea.
    4. The repository address, branch, and read-only Git credentials are configured during ThothII installation. This installation reads {repositoryLabel} on branch {statusQuery.data?.branch ?? "main"}.
    5. ThothII fetches the configured branch into its managed read-only checkout, validates the complete candidate revision, and activates it only when validation succeeds. It never edits, commits, pushes, or publishes workspace source.
      6. @@ -337,21 +358,56 @@ export function WorkspaceManager({ open, onClose }: { open: boolean; onClose: ()
      Source file
      {selectedSummary.file}
      Active revision
      {detailQuery.data.revision.commit}
      -
      Data warehouse
      {detailQuery.data.workspace.dwh.engine} · {detailQuery.data.workspace.dwh.database}/{detailQuery.data.workspace.dwh.schema}
      +
      +
      Database
      +
      + engine: {detailQuery.data.workspace.dwh.engine} + database: {detailQuery.data.workspace.dwh.database} + schema: {detailQuery.data.workspace.dwh.schema} +
      +
      Runtime status
      {stateLabel(runtime.configurationState)}
      -
      +

      Validate workspace source

      Checks workspace.yaml and the required workspace directories against the supported workspace schema. No source file is changed.

      + {validationNotice && ( +

      + {validationNotice} +

      + )} + {validationDiagnostics.length > 0 && ( +
      + {validationDiagnostics.map((diagnostic) => ( +

      + {diagnostic} +

      + ))} +
      + )}
      -
      +

      Test workspace connections

      Uses temporary decrypted credentials to verify the configured data warehouse and Evidence source. Temporary files are deleted after the test.

      + {connectionNotice && ( +

      + {connectionNotice} +

      + )} + {connectionDiagnostics.length > 0 && ( +
      + {connectionDiagnostics.map((diagnostic) => ( +

      + {diagnostic} +

      + ))} +
      + )} diff --git a/tools/tht/internal/output/sanitize.go b/tools/tht/internal/output/sanitize.go index 6b2a4c9d..cab30404 100644 --- a/tools/tht/internal/output/sanitize.go +++ b/tools/tht/internal/output/sanitize.go @@ -122,6 +122,12 @@ func extractSecretValues(contents []byte) ([]string, error) { if whole != "" { values = append(values, whole) } + // PEM files (including OpenSSH private keys) can contain base64 lines that look + // like dotenv assignments. Keep the complete document opaque instead of trying + // to parse it as a dotenv bundle. + if bytes.HasPrefix(trimmed, []byte("-----BEGIN ")) { + return values, nil + } if trimmed[0] == '{' || trimmed[0] == '[' { var document any diff --git a/tools/tht/internal/output/sanitize_test.go b/tools/tht/internal/output/sanitize_test.go index e96f666f..ee375729 100644 --- a/tools/tht/internal/output/sanitize_test.go +++ b/tools/tht/internal/output/sanitize_test.go @@ -105,6 +105,26 @@ func TestSecretValuesFromFilesRedactsEveryDotenvBundleValue(t *testing.T) { } } +func TestSecretValuesFromFilesAcceptsOpenSSHPrivateKey(t *testing.T) { + t.Parallel() + + secretFile := filepath.Join(physicalTempDir(t), "git-ssh-key") + contents := "-----BEGIN OPENSSH PRIVATE KEY-----\n" + + "ZmFrZS1rZXktcGF5bG9hZA==\n" + + "-----END OPENSSH PRIVATE KEY-----\n" + if err := os.WriteFile(secretFile, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + + secrets, err := SecretValuesFromFiles([]string{secretFile}) + if err != nil { + t.Fatalf("SecretValuesFromFiles() error = %v, want OpenSSH key accepted", err) + } + if len(secrets) == 0 { + t.Fatal("SecretValuesFromFiles() returned no values for OpenSSH key") + } +} + func TestSanitizeRecognizesQuotedCredentialKeys(t *testing.T) { t.Parallel() diff --git a/tools/tht/internal/pi/commands.go b/tools/tht/internal/pi/commands.go index 0be67c6f..def58e40 100644 --- a/tools/tht/internal/pi/commands.go +++ b/tools/tht/internal/pi/commands.go @@ -42,7 +42,7 @@ type settingsFileSnapshot struct { var internalIdentityHeaders = []string{ "-H", "x-thoth-principal-issuer: tht", "-H", "x-thoth-principal-subject: tht-maintenance", - "-H", "x-thoth-principal-display-name: Tht maintenance", + "-H", "x-thoth-principal-display-name: Tht maintenance", "-H", "x-thoth-is-admin: 1", } diff --git a/tools/tht/internal/pi/version.go b/tools/tht/internal/pi/version.go new file mode 100644 index 00000000..3eefdac4 --- /dev/null +++ b/tools/tht/internal/pi/version.go @@ -0,0 +1,46 @@ +package pi + +import ( + "bufio" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +var defaultPiArgPattern = regexp.MustCompile(`^ARG[[:space:]]+PI_VERSION[[:space:]]*=(.*)$`) + +// ReadPinnedVersion reads the repository's declared Pi runtime version. It deliberately reads +// only the default ARG, not the later ARG PI_VERSION declarations used by build stages. +func ReadPinnedVersion(projectDirectory string) (string, error) { + path := filepath.Join(projectDirectory, "docker", "core.Dockerfile") + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("read Pi version pin: %w", err) + } + defer file.Close() + + var version string + count := 0 + scanner := bufio.NewScanner(file) + for scanner.Scan() { + match := defaultPiArgPattern.FindStringSubmatch(strings.TrimSuffix(scanner.Text(), "\r")) + if len(match) != 2 { + continue + } + count++ + version = strings.TrimSpace(match[1]) + } + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("read Pi version pin: %w", err) + } + if count != 1 { + return "", errors.New("docker/core.Dockerfile must contain exactly one default PI_VERSION") + } + if _, err := parseSemanticVersion(version); err != nil { + return "", errors.New("docker/core.Dockerfile contains an invalid Pi version") + } + return version, nil +} diff --git a/tools/tht/internal/pi/version_test.go b/tools/tht/internal/pi/version_test.go new file mode 100644 index 00000000..45f285eb --- /dev/null +++ b/tools/tht/internal/pi/version_test.go @@ -0,0 +1,52 @@ +package pi + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadPinnedVersionReadsTheSingleDefaultPiArg(t *testing.T) { + root := t.TempDir() + writeCoreDockerfile(t, root, "ARG PI_VERSION=0.81.0\nARG PI_VERSION\n") + + got, err := ReadPinnedVersion(root) + if err != nil { + t.Fatal(err) + } + if got != "0.81.0" { + t.Fatalf("ReadPinnedVersion() = %q, want 0.81.0", got) + } +} + +func TestReadPinnedVersionRejectsMissingDuplicateAndMalformedPins(t *testing.T) { + for _, test := range []struct { + name string + file string + want string + }{ + {name: "missing", file: "# no default\n", want: "one default PI_VERSION"}, + {name: "duplicate", file: "ARG PI_VERSION=0.80.3\nARG PI_VERSION=0.81.0\n", want: "one default PI_VERSION"}, + {name: "malformed", file: "ARG PI_VERSION=latest\n", want: "invalid Pi version"}, + } { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + writeCoreDockerfile(t, root, test.file) + _, err := ReadPinnedVersion(root) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ReadPinnedVersion() error = %v, want %q", err, test.want) + } + }) + } +} + +func writeCoreDockerfile(t *testing.T, root, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "docker"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "docker", "core.Dockerfile"), []byte(contents), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/tools/tht/internal/workspaceops/operations.go b/tools/tht/internal/workspaceops/operations.go index 50c5e2ce..ec4645fc 100644 --- a/tools/tht/internal/workspaceops/operations.go +++ b/tools/tht/internal/workspaceops/operations.go @@ -90,14 +90,14 @@ type RunRequest struct { Resume string } -func (InspectRequest) workspaceRequest() {} -func (DwhRequest) workspaceRequest() {} -func (SuggestFksRequest) workspaceRequest() {} -func (CheckSchemaRequest) workspaceRequest() {} +func (InspectRequest) workspaceRequest() {} +func (DwhRequest) workspaceRequest() {} +func (SuggestFksRequest) workspaceRequest() {} +func (CheckSchemaRequest) workspaceRequest() {} func (AcceptSchemaRequest) workspaceRequest() {} -func (IndexSchemaRequest) workspaceRequest() {} -func (EvidenceRequest) workspaceRequest() {} -func (RunRequest) workspaceRequest() {} +func (IndexSchemaRequest) workspaceRequest() {} +func (EvidenceRequest) workspaceRequest() {} +func (RunRequest) workspaceRequest() {} func (InspectRequest) operatorCommand() string { return "inspect" } func (DwhRequest) operatorCommand() string { return "preprocess-dwh" } @@ -106,9 +106,9 @@ func (CheckSchemaRequest) operatorCommand() string { return "schema-check" } func (AcceptSchemaRequest) operatorCommand() string { return "schema-accept" } -func (IndexSchemaRequest) operatorCommand() string { return "index-schema" } -func (EvidenceRequest) operatorCommand() string { return "preprocess-evidence" } -func (RunRequest) operatorCommand() string { return "preprocess-run" } +func (IndexSchemaRequest) operatorCommand() string { return "index-schema" } +func (EvidenceRequest) operatorCommand() string { return "preprocess-evidence" } +func (RunRequest) operatorCommand() string { return "preprocess-run" } func (r InspectRequest) stdinEnvelope() (requestEnvelope, error) { return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace}, nil @@ -165,19 +165,19 @@ func (r RunRequest) stdinEnvelope() (requestEnvelope, error) { } type requestEnvelope struct { - SchemaVersion int `json:"schemaVersion"` - WorkspaceID string `json:"workspaceId"` - Resume string `json:"resumeRunId,omitempty"` - DryRun bool `json:"dryRun,omitempty"` - Assume []string `json:"assume,omitempty"` - SQLFiles []inputFile `json:"fromSql,omitempty"` - Annotations string `json:"annotationsYaml,omitempty"` - ReviewedCandidates string `json:"reviewedCandidatesDigest,omitempty"` - Collection string `json:"collection,omitempty"` - Confirm string `json:"confirm,omitempty"` - Destroy bool `json:"destroy,omitempty"` - RunID string `json:"runId,omitempty"` - Yes bool `json:"yes,omitempty"` + SchemaVersion int `json:"schemaVersion"` + WorkspaceID string `json:"workspaceId"` + Resume string `json:"resumeRunId,omitempty"` + DryRun bool `json:"dryRun,omitempty"` + Assume []string `json:"assume,omitempty"` + SQLFiles []inputFile `json:"fromSql,omitempty"` + Annotations string `json:"annotationsYaml,omitempty"` + ReviewedCandidates string `json:"reviewedCandidatesDigest,omitempty"` + Collection string `json:"collection,omitempty"` + Confirm string `json:"confirm,omitempty"` + Destroy bool `json:"destroy,omitempty"` + RunID string `json:"runId,omitempty"` + Yes bool `json:"yes,omitempty"` } type inputFile struct { @@ -186,23 +186,23 @@ type inputFile struct { } type Result struct { - SchemaVersion int `json:"schemaVersion"` - Status string `json:"status"` - Code string `json:"code"` - WorkspaceID string `json:"workspaceId"` - WorkspaceRevision string `json:"workspaceRevision"` - DescriptorBlob string `json:"descriptorBlob"` - Operation string `json:"operation"` - RunID string `json:"runId,omitempty"` - ChildRuns map[string]string `json:"childRuns,omitempty"` - CompletedStages []string `json:"completedStages"` - Counts map[string]int `json:"counts,omitempty"` - ArtifactIdentities []ArtifactIdentity `json:"artifactIdentities,omitempty"` - SuggestedFksYAML string `json:"suggestedFksYaml,omitempty"` - EffectiveConfigIdentity string `json:"effectiveConfigIdentity,omitempty"` - ConfigFingerprint string `json:"configFingerprint,omitempty"` - InputFingerprint string `json:"inputFingerprint,omitempty"` - Warnings []string `json:"warnings,omitempty"` + SchemaVersion int `json:"schemaVersion"` + Status string `json:"status"` + Code string `json:"code"` + WorkspaceID string `json:"workspaceId"` + WorkspaceRevision string `json:"workspaceRevision"` + DescriptorBlob string `json:"descriptorBlob"` + Operation string `json:"operation"` + RunID string `json:"runId,omitempty"` + ChildRuns map[string]string `json:"childRuns,omitempty"` + CompletedStages []string `json:"completedStages"` + Counts map[string]int `json:"counts,omitempty"` + ArtifactIdentities []ArtifactIdentity `json:"artifactIdentities,omitempty"` + SuggestedFksYAML string `json:"suggestedFksYaml,omitempty"` + EffectiveConfigIdentity string `json:"effectiveConfigIdentity,omitempty"` + ConfigFingerprint string `json:"configFingerprint,omitempty"` + InputFingerprint string `json:"inputFingerprint,omitempty"` + Warnings []string `json:"warnings,omitempty"` } type ArtifactIdentity struct { @@ -883,8 +883,8 @@ type VectorRebuildRequest struct { Destroy bool } -func (VectorInspectRequest) workspaceRequest() {} -func (VectorRebuildRequest) workspaceRequest() {} +func (VectorInspectRequest) workspaceRequest() {} +func (VectorRebuildRequest) workspaceRequest() {} func (VectorInspectRequest) operatorCommand() string { return "vector-inspect" } func (VectorRebuildRequest) operatorCommand() string { return "vector-rebuild" } func (r VectorInspectRequest) stdinEnvelope() (requestEnvelope, error) {