Compare commits
54
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ec0421e9fd | ||
|
|
55f3569e55 | ||
|
|
b9c3369e7b | ||
|
|
64e6b9664a | ||
|
|
67ee52624c | ||
|
|
0d2e573e0d | ||
|
|
bd416f7327 | ||
|
|
23e52c80de | ||
|
|
84084bba37 | ||
|
|
efd7d788d9 | ||
|
|
b1c510a097 | ||
|
|
5f3680a0fb | ||
|
|
4ff91e8d6e | ||
|
|
5f3a7f5975 | ||
|
|
043ffdfad6 | ||
|
|
6a4634dcf1 | ||
|
|
84804be9f8 | ||
|
|
c3caba94dd | ||
|
|
b1723c34c4 | ||
|
|
d6cdffea62 | ||
|
|
49333a2d35 | ||
|
|
b006b94479 | ||
|
|
bdcd8fcd28 | ||
|
|
cf90c1bd51 | ||
|
|
571a4bcaa2 | ||
|
|
9051463654 | ||
|
|
26c5605ff7 | ||
|
|
3535fda958 | ||
|
|
2953f6b608 | ||
|
|
45db3a239b | ||
|
|
7d826e46c0 | ||
|
|
648434a32e | ||
|
|
023b822f83 | ||
|
|
d8a29bfbdd | ||
|
|
a59624a68f | ||
|
|
5af4408194 | ||
|
|
e088abd60a | ||
|
|
803e9e9201 | ||
|
|
8c81996896 | ||
|
|
eed398e569 | ||
|
|
c8d276ddc6 | ||
|
|
2d1b714ebe | ||
|
|
c7e5f295e6 | ||
|
|
f52bf22e05 | ||
|
|
840344706f | ||
|
|
41b9fed4d5 | ||
|
|
36bf659ea9 | ||
|
|
4a67d60233 | ||
|
|
debb63d87b | ||
|
|
3943022a97 | ||
|
|
f5ec2d9313 | ||
|
|
82e2c91f42 | ||
|
|
8fe526dd6e | ||
|
|
e68e80a33d |
@@ -17,6 +17,7 @@ frontend/vite.database-management-prototype.config.ts
|
||||
!deploy/env/*.env.example
|
||||
deploy/thothii.env
|
||||
deploy/secrets/
|
||||
deploy/psd/
|
||||
harness/workspaces/*.yaml
|
||||
!harness/workspaces/local.yaml
|
||||
!harness/workspaces/tht.example.yaml
|
||||
@@ -29,3 +30,7 @@ coverage/
|
||||
data/
|
||||
sessions/
|
||||
workspace-registry/
|
||||
|
||||
.tht/
|
||||
|
||||
deploy/local/
|
||||
|
||||
@@ -7,6 +7,11 @@ on:
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "mkdocs.yml"
|
||||
- "scripts/build-docs.sh"
|
||||
- "scripts/verify-public-docs.py"
|
||||
- "scripts/test-verify-public-docs.py"
|
||||
- "scripts/verify-auth-docs.py"
|
||||
- "scripts/test-verify-auth-docs.py"
|
||||
- "docs/requirements.txt"
|
||||
- ".gitea/workflows/publish-docs.yml"
|
||||
workflow_dispatch:
|
||||
@@ -34,14 +39,25 @@ jobs:
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: docs/requirements.txt
|
||||
cache-dependency-path: docs/requirements.lock
|
||||
|
||||
- name: Install MkDocs dependencies
|
||||
run: python -m pip install -r docs/requirements.txt
|
||||
run: python -m pip install -r docs/requirements.lock
|
||||
|
||||
- name: Test public documentation boundary
|
||||
run: python scripts/test-verify-public-docs.py
|
||||
|
||||
- name: Test current authentication documentation
|
||||
run: |
|
||||
python scripts/verify-auth-docs.py auth
|
||||
python scripts/verify-auth-docs.py dwh
|
||||
python scripts/test-verify-auth-docs.py auth
|
||||
python scripts/test-verify-auth-docs.py dwh
|
||||
|
||||
- name: Build documentation
|
||||
# Some documented source files intentionally live outside docs/.
|
||||
run: mkdocs build
|
||||
run: |
|
||||
mkdocs build --strict
|
||||
python scripts/verify-public-docs.py
|
||||
|
||||
- name: Publish generated site to the pages branch
|
||||
working-directory: site
|
||||
|
||||
@@ -46,6 +46,7 @@ deploy/secrets/*
|
||||
# Per-installation configuration generated by `tht setup` (examples stay tracked).
|
||||
deploy/*/thothii-installation.yaml
|
||||
deploy/*/operator.env
|
||||
deploy/*/auth/
|
||||
deploy/*/generated/
|
||||
deploy/*/secrets/*
|
||||
!deploy/*/secrets/.gitkeep
|
||||
|
||||
@@ -98,8 +98,18 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
|
||||
- **`tht`'s `-c`/`--config` is a PER-COMMAND option** — it must follow the subcommand, never
|
||||
precede it (`ThtRunner.buildArgv` enforces this; prepending caused live 500s).
|
||||
- **`--json` output must be pristine** (only valid JSON on stdout) — used as a machine contract.
|
||||
- **UI strings are English; document *content* stays the workspace language** (Italian for
|
||||
`psd`) because it's the real data. Only chrome/labels are English.
|
||||
- **Localization:** deterministic UI uses the EN/IT catalogs with English fallback;
|
||||
model interaction uses the session manifest's immutable `interaction_language`.
|
||||
Workspace content, SQL, and identifiers remain unchanged. For shell modes, portal
|
||||
integration, or translations, read `docs/operations/shell-and-localization.md`.
|
||||
- **Server deployment:** for the coordinated ThothII/Omics upgrade, follow
|
||||
`docs/operations/server-codex-handoff.md`; it supersedes earlier Omics delivery
|
||||
instructions. Omics source integration uses GitHub with no repository relay prerequisite.
|
||||
- **Server identity:** for portal login/logout, proxy headers, or Omics deploy, read
|
||||
`docs/install/authentication-upstream.md` before changing authentication. Omics
|
||||
uses embedded/upstream, not a second ThothII OIDC login. Full/embedded rendering
|
||||
is documented in `docs/architecture/application-shell.md`; release acceptance
|
||||
is in `docs/testing/authentication-manual-acceptance.md`.
|
||||
- **Workspace schema v4** defines workspace identity and optional Evidence only. PostgreSQL Metadata
|
||||
Catalog owns database identity, binding, schema, descriptions, sensitivity, and relationships;
|
||||
embedding/model facts come from the installation catalog. The legacy `harness/workspaces/*.yaml` runtime snapshots still use
|
||||
|
||||
+158
-12
@@ -91,21 +91,79 @@ correzione successiva crea una nuova sessione derivata, collegata a quella prece
|
||||
dopo la finalizzazione. Non può modificare il ledger, gli artifact canonici o lo stato
|
||||
terminale della sessione.
|
||||
|
||||
## Memory
|
||||
|
||||
**Memory Module** — Il modulo che possiede le conoscenze ed esperienze curate per
|
||||
migliorare schema linking e generazione SQL di domande future. Le Memory appartengono
|
||||
a un workspace e rimangono distinte dalle Evidence.
|
||||
|
||||
**Memory Card** — L'unità di contenuto gestibile del Memory Module, con identità,
|
||||
ambito di applicazione e provenienza. Il formato è allineato per analogia alle
|
||||
Evidence, senza implicare la stessa origine o lo stesso percorso di pubblicazione.
|
||||
|
||||
**Reusable Memory** — Una Memory Card che esprime un chiarimento di dominio, una
|
||||
regola di costruzione SQL o un errore da evitare con motivo compreso e approvato.
|
||||
La sua validità è circoscritta a un ambito esplicito e non deriva dalla sola
|
||||
approvazione di una scelta occasionale in una domanda.
|
||||
|
||||
**Solved Question** — Una Memory Card che conserva una domanda risolta con la
|
||||
relativa soluzione SQL e il contesto necessario a interpretarla. È un exemplar
|
||||
consultativo: i parametri e le scelte del caso non diventano regole generali.
|
||||
|
||||
**Memory Graph** — L'insieme dei collegamenti espliciti fra card che contribuisce
|
||||
al recupero di conoscenze pertinenti oltre alla somiglianza del contenuto. Il
|
||||
ritrovamento di una card tramite un collegamento non ne implica l'approvazione.
|
||||
|
||||
**Memory Link** — Un collegamento curato fra card, con destinazione e significato
|
||||
espliciti, che contribuisce alla consultazione di contenuti pertinenti. La sua
|
||||
rimozione non comporta la cancellazione delle card collegate.
|
||||
|
||||
## Evidence
|
||||
|
||||
**Context specialist** — La persona competente sul dominio che redige e cura il
|
||||
contenuto delle Evidence. Può essere distinta da chi amministra l'installazione;
|
||||
il suo lavoro di redazione non richiede accesso al database applicativo.
|
||||
|
||||
**Evidence draft** — Il documento iniziale scritto dallo specialista di contesto,
|
||||
che il sistema acquisisce e raffina in Evidence Unit. Può essere redatto e
|
||||
consegnato indipendentemente dall'installazione che userà le Evidence risultanti.
|
||||
|
||||
**Evidence Module** — Il modulo autonomo che possiede la preparazione delle Evidence e
|
||||
la loro consultazione durante il workflow. La preparazione avviene fuori dalle singole
|
||||
sessioni; il workflow usa soltanto contenuti già pubblicati. A runtime contribuisce agli
|
||||
stage semantici esistenti, senza diventare uno stage visibile e senza modificare ledger,
|
||||
artifact o stato del workflow.
|
||||
|
||||
**Source Evidence** — Un documento originale del workspace, conservato senza modifiche
|
||||
come riferimento umano e origine della successiva ristrutturazione.
|
||||
**Source Evidence** — Il documento o la dichiarazione che sostiene il contenuto
|
||||
corrente di una Evidence Unit. Un documento acquisito viene conservato come
|
||||
riferimento umano; una dichiarazione manuale attribuisce il contenuto alla persona
|
||||
che lo ha scritto e approvato.
|
||||
|
||||
**Manual Evidence declaration** — Una dichiarazione esplicita dell'amministratore
|
||||
che sostiene una Evidence creata direttamente o una correzione del suo significato.
|
||||
Non implica una verifica indipendente da parte di una fonte documentale esterna.
|
||||
|
||||
**Evidence origin** — Il documento da cui una Evidence Unit è stata inizialmente
|
||||
derivata. Può restare collegato per provenienza e confronto con gli aggiornamenti
|
||||
anche quando una dichiarazione manuale sostiene il testo corrente. La sola origine
|
||||
non dimostra il supporto semantico di una successiva correzione.
|
||||
|
||||
**Local Evidence archive** — L'insieme delle Evidence curate custodite
|
||||
dall'installazione, distinto dalle draft originali e dai contenuti derivati per
|
||||
la ricerca. Comprende le correzioni manuali e i ritiri deliberati.
|
||||
|
||||
**Consolidated Evidence** — Una versione delle Evidence locali controllata come
|
||||
insieme coerente e pronta per l'attivazione. I file ancora in modifica non ne
|
||||
cambiano il contenuto.
|
||||
|
||||
**Active Evidence** — La versione consolidata disponibile alla consultazione del
|
||||
core. Un tentativo di aggiornamento fallito conserva la versione attiva precedente.
|
||||
|
||||
**Evidence Unit** — La più piccola unità semantica coerente, revisionabile e ricercabile
|
||||
derivata da una sola Source Evidence. Possiede un identificatore stabile indipendente
|
||||
dal kind, assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono
|
||||
fuse automaticamente.
|
||||
fondata su una Source Evidence corrente, anche manuale, e con eventuale origine
|
||||
documentale distinta. Possiede un identificatore stabile indipendente dal kind,
|
||||
assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono fuse
|
||||
automaticamente.
|
||||
|
||||
**Evidence kind** — La categoria semantica di una Evidence Unit, che ne determina i
|
||||
campi specifici e ne orienta l'uso. Ogni unità ha un solo kind primario; i tipi iniziali
|
||||
@@ -151,10 +209,9 @@ avanzare fino a un retry riuscito.
|
||||
nella sessione: stage semantico, purpose, generazione interrogata e identificatori delle
|
||||
Evidence restituite. Non duplica il contenuto delle Evidence.
|
||||
|
||||
**Curated Evidence** — Una o più Evidence Unit ristrutturate a partire da una Source
|
||||
Evidence e conservate nel repository del workspace come proposte per la revisione
|
||||
umana. Git conserva la versione precedente e rende visibile ogni modifica; una Curated
|
||||
Evidence non è ancora contenuto autorevole del runtime.
|
||||
**Curated Evidence** — Una o più Evidence Unit preparate da documenti o curate
|
||||
manualmente. La presenza nell'archivio curato non implica da sola che il contenuto
|
||||
sia già attivo per il workflow.
|
||||
|
||||
**Published Evidence** — Le Curated Evidence valide appartenenti alla revisione attiva
|
||||
del workspace e alla generazione Evidence pubblicata. L'approvazione umana precede
|
||||
@@ -176,9 +233,17 @@ una Evidence Unit. Il sistema ne verifica deterministicamente la presenza dopo l
|
||||
normalizzazione meccanica; il curatore resta responsabile di verificarne la sufficienza
|
||||
semantica.
|
||||
|
||||
**Evidence resolution** — L'operazione esplicita con cui un curatore ritira una
|
||||
Evidence Unit oppure la ricollega a un Source Evidence esistente. Aggiorna documento e
|
||||
manifest insieme, lascia un diff Git revisionabile e non pubblica né crea commit.
|
||||
**Evidence resolution** — La decisione esplicita con cui un curatore risolve un
|
||||
problema di una Evidence Unit, correggendola, ritirandola oppure ricollegandola a
|
||||
una fonte adeguata.
|
||||
|
||||
**Source update conflict** — Un contrasto fra una fonte aggiornata e una correzione
|
||||
manuale già approvata. La correzione resta in uso fino alla risoluzione esplicita
|
||||
del confronto da parte dell'amministratore.
|
||||
|
||||
**Evidence source refresh** — La riacquisizione delle fonti esterne richiesta
|
||||
dall'amministratore per rilevarne le modifiche. Fra due aggiornamenti il contenuto
|
||||
già acquisito resta il riferimento per preparazione e consultazione.
|
||||
|
||||
**Review item** — Un blocco di revisione descritto da codice stabile, messaggio umano e
|
||||
campo opzionale. Finché viene mantenuto nell'Evidence Unit, ne impedisce la
|
||||
@@ -506,3 +571,84 @@ _Avoid_: Sensitive Data Suggestion Event
|
||||
**Introspection Capability** — Una categoria di struttura fisica che una Database Binding
|
||||
può osservare, come tabelle, colonne, relazioni, indici o enum. Una capability non disponibile
|
||||
è distinta da una capability osservata che non ha restituito elementi.
|
||||
|
||||
## Amministrazione e integrazione
|
||||
|
||||
**Workspace Readiness** — La preparazione di uno specifico Workspace per l'uso nel
|
||||
workflow, comprensiva della disponibilità degli artefatti derivati dai suoi metadati
|
||||
Database e dalle sue Evidence. Il preprocessing appartiene a questa preparazione;
|
||||
la configurazione e la sincronizzazione del catalogo restano responsabilità Database.
|
||||
|
||||
**Administration Surface** — Una superficie amministrativa autonoma per configurare o curare una
|
||||
parte dell'installazione. Workspace, Evidence, Memory, Database e Pi sono superfici peer e non
|
||||
dipendono dall'esistenza di una sessione attiva.
|
||||
|
||||
**Administration Page** — La rappresentazione a pagina intera di una Administration Surface, con
|
||||
una gerarchia condivisa per identità, stato, azioni e contenuto. Un form amministrativo appartiene
|
||||
alla pagina e non a una popup come contenitore principale.
|
||||
_Avoid_: management popup, settings modal
|
||||
|
||||
**Administration Route** — L'identità navigabile di una Administration Surface nel browser. Deve
|
||||
essere ripristinabile con refresh e cronologia e non contiene valori transitori o segreti dei form.
|
||||
|
||||
**Embedded Thoth Shell** — L'esperienza Thoth ospitata dentro il documento e il contesto visuale di
|
||||
un portale host. Conserva la propria gerarchia funzionale, ma deve rispettare la geometria,
|
||||
l'autenticazione e le regole responsive del portale host.
|
||||
|
||||
**Full Thoth Shell** — L'esperienza Thoth autonoma che possiede il proprio header e il proprio
|
||||
layout di pagina. Non replica la navigazione amministrativa del portale host e non dipende dal suo
|
||||
template visuale.
|
||||
|
||||
**Shell mode** — La scelta di installazione fra `embedded` e `full`. Determina chi possiede il
|
||||
chrome globale, i comandi di identità e le integrazioni visuali, ma non cambia il workflow o la
|
||||
persistenza delle sessioni.
|
||||
|
||||
**Fullscreen state** — Lo stato temporaneo in cui il documento applicativo occupa il fullscreen
|
||||
del browser. È distinto da `Shell mode`: una Full Thoth Shell può essere aperta senza fullscreen;
|
||||
il passaggio è attivato da un comando esplicito e può essere annullato con la stessa azione o con
|
||||
il comando nativo del browser.
|
||||
|
||||
**Portal Shell Adapter** — Il confine sostituibile che traduce lo stato e i comandi del chrome di
|
||||
un portale host nel modello semantico usato da Thoth. L'adapter non possiede autorizzazione,
|
||||
sessioni di workflow o contenuti del modello.
|
||||
|
||||
**Host Shell State** — Il minimo stato visuale fornito dal portale host: locale UI, tema e stato
|
||||
fullscreen. In una Embedded Thoth Shell è la fonte autorevole per queste preferenze;
|
||||
non include identità, token o stato di autenticazione, che restano responsabilità dell'accesso.
|
||||
|
||||
**UI locale** — La lingua delle label, dei messaggi, dei tooltip, degli stati e delle istruzioni
|
||||
non generate dal modello nell'interfaccia Thoth. È distinta dalla lingua dei contenuti di un
|
||||
workspace.
|
||||
|
||||
**Interaction language** — La lingua in cui il modello presenta domande, spiegazioni e proposte
|
||||
al revisore durante una sessione. Viene fissata alla creazione della sessione e rimane invariata
|
||||
durante una ripresa, anche se la UI locale corrente cambia.
|
||||
|
||||
**Administrative Page Family** — L'insieme delle cinque Administration Page che condividono shell,
|
||||
navigazione, tipografia e regole responsive, pur mantenendo contenuti e operazioni specifici:
|
||||
Workspace, Evidence, Memory, Database e Pi.
|
||||
|
||||
## Installazione
|
||||
|
||||
**Manual standalone installation** — Una copia di ThothII predisposta per l'uso autonomo da una
|
||||
persona che possiede il computer, con una Full Thoth Shell e servizi applicativi locali. La
|
||||
procedura non implica che DWH o provider LLM siano locali o disponibili offline.
|
||||
|
||||
**Installation bootstrap** — L'insieme delle attività iniziali che rende disponibile una
|
||||
installazione manuale: verifica dell'host, generazione della configurazione, predisposizione
|
||||
delle credenziali protette e avvio dei servizi. Non è un installer dell'applicazione.
|
||||
|
||||
**Installation preparation** — La predisposizione dei documenti che descrivono i workspace
|
||||
e i parametri dell'installazione, prima di applicarli. Permette all'operatore di raccogliere
|
||||
e correggere le informazioni senza avviare l'applicazione.
|
||||
|
||||
**Installation validation** — La verifica ripetibile della completezza e coerenza dei
|
||||
documenti e delle precondizioni di un'installazione. Distingue ciò che è stato verificato
|
||||
da ciò che richiede un'applicazione già avviata.
|
||||
|
||||
**Installation execution** — L'applicazione dei documenti verificati per predisporre e
|
||||
avviare ThothII. Non raccoglie nuovi parametri dall'operatore durante l'esecuzione.
|
||||
|
||||
**Platform acceptance** — La verifica che una Manual standalone installation possa essere
|
||||
predisposta e avviata su una specifica combinazione di sistema operativo, architettura e runtime,
|
||||
distinta dalla verifica funzionale del collegamento a DWH e provider LLM.
|
||||
|
||||
@@ -11,50 +11,50 @@ colors:
|
||||
warm-graphite: "oklch(26.78% 0.0097 355.6)"
|
||||
muted-graphite: "oklch(51.33% 0.0088 345.6)"
|
||||
quiet-border: "oklch(90.93% 0.0035 354.7)"
|
||||
success-mint: "oklch(75.77% 0.1581 165)"
|
||||
success-mint: "oklch(46% 0.095 160)"
|
||||
navigation-active: "oklch(92.5% 0.052 23.2)"
|
||||
navigation-active-hover: "oklch(89.5% 0.071 23.2)"
|
||||
navigation-active-foreground: "oklch(36.5% 0.11 23.2)"
|
||||
navigation-active-border: "oklch(60% 0.135 23.2)"
|
||||
warning-amber: "oklch(85.23% 0.1386 78.9)"
|
||||
information-blue: "oklch(70.35% 0.1128 221.3)"
|
||||
warning-amber: "oklch(48% 0.09 70)"
|
||||
information-neutral: "oklch(51.33% 0.0088 345.6)"
|
||||
typography:
|
||||
display:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "3rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.5rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.03
|
||||
letterSpacing: "-0.025em"
|
||||
headline:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "1.875rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.5rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.15
|
||||
letterSpacing: "-0.015em"
|
||||
title:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "1.2rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.25rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "-0.01em"
|
||||
body:
|
||||
fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "0.9375rem"
|
||||
fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "1rem"
|
||||
fontWeight: 400
|
||||
lineHeight: 1.65
|
||||
letterSpacing: "normal"
|
||||
control:
|
||||
fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "0.875rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "0.005em"
|
||||
label:
|
||||
fontFamily: "ui-monospace, SF Mono, Cascadia Code, Menlo, Consolas, monospace"
|
||||
fontSize: "0.6875rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "0.75rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "0.06em"
|
||||
letterSpacing: "normal"
|
||||
rounded:
|
||||
xs: "4px"
|
||||
sm: "6px"
|
||||
@@ -113,6 +113,16 @@ components:
|
||||
|
||||
# Design System: ThothII
|
||||
|
||||
## Visual review branch, September 2026
|
||||
|
||||
The revision on `codex/ui-visual-review` is approved for implementation and Docker visual review,
|
||||
not yet for adoption on `main`. The previous look remains recoverable from the base commit and
|
||||
the preserved Docker image. Historical prototypes must remain untouched.
|
||||
|
||||
This revision follows Impeccable's product register: one locally bundled Manrope family for the
|
||||
whole UI, five fixed size roles, red as the sole brand accent and additional color only for meaningful
|
||||
state. The primary scene remains an analyst reading data and SQL in a well-lit office.
|
||||
|
||||
## Overview
|
||||
|
||||
**Creative North Star: "The Clinical Workbench"**
|
||||
@@ -134,7 +144,7 @@ disciplined and tactile, never playful, sluggish, or visually unstable.
|
||||
**Key Characteristics:**
|
||||
|
||||
- Warm, restrained surfaces with one scarce red accent.
|
||||
- Editorial headings paired with highly legible operational body text.
|
||||
- One sans-serif family, with hierarchy expressed through size, weight and spacing.
|
||||
- Dense information organized through hierarchy, rhythm, and progressive disclosure.
|
||||
- Persisted artifacts and reviewer decisions presented as the visual source of truth.
|
||||
- Fast state feedback with reduced-motion parity.
|
||||
@@ -150,6 +160,15 @@ users can scan structure without adding nested containers.
|
||||
|
||||
## Colors
|
||||
|
||||
The full-mode application header matches Omics Portal's `--gsd-red-primary`
|
||||
(`#CB333B`) in both themes. Its complete wordmark, including `II`, and controls
|
||||
use a near-white foreground. This header is absent in embedded mode. The sidebar
|
||||
and welcome wordmarks retain their red suffix. Context editing places workspace,
|
||||
model and Done in one desktop row, stacking on narrow containers. Session-scope
|
||||
tabs retain their selected fill and accessible keyboard state with a uniform one-pixel
|
||||
border on every side, gray when inactive and red when active. Their padding is 11px
|
||||
horizontal and 3px vertical, with a 38px minimum height and wrapping labels.
|
||||
|
||||
The palette combines warm porcelain surfaces, warm graphite text, and an instrument red used only
|
||||
for action, focus, and important state. OKLCH values in the frontmatter are normative because the
|
||||
frontend uses OKLCH tokens directly.
|
||||
@@ -178,7 +197,8 @@ frontend uses OKLCH tokens directly.
|
||||
foreground. It shares Instrument Red's hue but uses a lighter, lower-chroma fill, so location is
|
||||
visible without carrying the full weight of a primary action.
|
||||
- **Warning Amber** (`warning-amber`): waiting, attention, and in-progress states.
|
||||
- **Information Blue** (`information-blue`): informational state when red would imply action.
|
||||
- **Information**: neutral text and indicators for dates, protocols and ordinary status. The legacy
|
||||
`--info` token resolves to muted foreground, not an additional blue accent.
|
||||
|
||||
The dark theme keeps the same semantic mapping with neutral near-black surfaces and a slightly
|
||||
lighter red accent. Do not introduce a second visual identity for dark mode.
|
||||
@@ -191,30 +211,33 @@ for their named states. Color is never the only state indicator.
|
||||
|
||||
## Typography
|
||||
|
||||
**Display Font:** Fraunces, with Source Serif Pro, Georgia, and Times New Roman fallbacks
|
||||
**Body Font:** Manrope, with native system sans-serif fallbacks
|
||||
**Label/Mono Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks
|
||||
**UI Font:** locally bundled Manrope Variable, with Manrope and native sans-serif fallbacks.
|
||||
**Technical Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks.
|
||||
|
||||
**Character:** Fraunces gives persisted artifacts and key headings editorial authority. Manrope
|
||||
keeps dense controls and prose calm and readable. The mono register separates machine identity,
|
||||
metadata, SQL, identifiers, and micro-labels from natural-language content.
|
||||
Manrope covers headings, labels, controls, navigation and document reading. Monospace is reserved
|
||||
for SQL, code, paths and machine identifiers, never for ordinary UI labels or status headings.
|
||||
|
||||
### Hierarchy
|
||||
|
||||
- **Display** (600, `3rem`, `1.03`): authentication and exceptional page-level statements only.
|
||||
- **Headline** (600, `1.875rem`, `1.15`): major page or artifact titles.
|
||||
- **Title** (600, `1.2rem`, `1.25`): panel and document section hierarchy.
|
||||
- **Body** (400, `0.9375rem`, `1.65`): operational prose, with a target line length of 65 to 75
|
||||
- **Headline** (600, `1.5rem`, `1.3`): page or artifact titles, `--text-page`.
|
||||
- **Title** (600, `1.25rem`, `1.4`): section hierarchy, `--text-section`.
|
||||
- **Body** (400, `1rem`, `1.6`): operational prose, `--text-body`, with a target line length of 65 to 75
|
||||
characters where the surface controls width.
|
||||
- **Control** (600, `0.875rem`, `1.25`): buttons, inputs, tabs, and compact actions.
|
||||
- **Label** (600, `0.6875rem`, `0.06em` tracking): uppercase micro-labels, state metadata, and panel
|
||||
headers. Labels use the mono family.
|
||||
- **Control** (400–600, `0.875rem`, `1.5`): buttons, inputs, tables, tabs and compact subheadings,
|
||||
`--text-control`.
|
||||
- **Metadata** (400–600, `0.75rem`, `1.5`): secondary status, counts and timestamps, `--text-meta`.
|
||||
Labels use sentence case and normal tracking. Ordinary operational text never falls below 12px.
|
||||
|
||||
Typography uses fixed sizes. Responsive changes happen at structural breakpoints, not through fluid
|
||||
type scaling. Numeric data and identifiers use tabular numerals where comparison matters.
|
||||
|
||||
**The Three Registers Rule.** Serif means authority, sans means interaction and reading, mono means
|
||||
machine identity. Do not exchange these roles for novelty.
|
||||
**Application wordmark:** ThothII is a brand mark, not a page title: use Manrope semibold at
|
||||
48px (`3rem`) in the Core welcome area and 32px (`2rem`) in the session sidebar, with the
|
||||
`II` suffix in brand red. Preserve these sizes across responsive layouts.
|
||||
|
||||
**The One Family Rule.** The UI and document readers use sans-serif throughout. The legacy
|
||||
`--font-heading` alias resolves to `--font-sans`. Preserve technical monospace without turning it
|
||||
into a second decorative hierarchy. Do not shrink text to solve layout constraints.
|
||||
|
||||
**The Read Once Rule.** A heading, label, and body must be distinguishable on first glance through
|
||||
size and weight. Do not repeat headings in explanatory copy.
|
||||
@@ -279,27 +302,70 @@ default, hover, focus, active, disabled, loading, and error behavior where those
|
||||
- **Focus:** three-pixel Instrument Red ring with a clear border shift.
|
||||
- **Error / Disabled:** errors combine destructive color with explanatory text; disabled controls
|
||||
retain readable contrast and use 50 percent opacity.
|
||||
- **Metadata catalog model:** Database Management keeps one compact, installation-level
|
||||
metadata-generation LLM selector in the application header. The selection persists across
|
||||
database, table, column, and relationship views; when no usable profile is configured, the
|
||||
disabled control explains: “No metadata-generation LLM model is configured for this installation.”
|
||||
- **Global context:** the collapsible top shelf is the sole workspace/model selector for Core and
|
||||
Admin. Preserve independent remembered choices, installation defaults, operation locks and unsaved
|
||||
edit guards. Never introduce a separate metadata-generation default or selector.
|
||||
|
||||
### Navigation
|
||||
|
||||
- **Workspace readiness:** the Workspace navigation button carries an 8px dot to
|
||||
the right of its label. Green means a selected workspace with confirmed ready
|
||||
preprocessing and no query error; all other states are red. The button's
|
||||
tooltip and accessible description retain the translated exact state. Do not
|
||||
add a separate readiness text row or change the backend readiness gate.
|
||||
- **Session groups:** one accessible single-open accordion contains Active sessions
|
||||
and Archive, both initially closed. Below the scope tabs, show only their
|
||||
adjacent section headers, without a redundant Sessions heading. Selection and
|
||||
bulk-delete controls belong inside each panel and only appear for nonempty
|
||||
lists. Select all affects that list only, preserves the other list's selection,
|
||||
and exposes a mixed state for partial selection. Preserve the existing archived
|
||||
flag as the grouping rule, independent of whether a Pi process is running.
|
||||
Opening a section closes the other; either can be collapsed, including both.
|
||||
Empty lists show only the translated "No sessions yet." message.
|
||||
The open section uses the rail's remaining height; its list scrolls internally
|
||||
with a cap of `min(18rem, 35dvh)`, while its trigger remains outside that scroll
|
||||
area. The mobile navigation dialog supplies a bounded viewport-height container.
|
||||
Keyboard users can focus and scroll each labelled panel.
|
||||
- **Session entry:** one Session button returns to the current unfinished session,
|
||||
including provisional creation, without resetting or reconnecting it. Otherwise
|
||||
it prepares a new question using the normal readiness and unsaved-work guards.
|
||||
- **Style:** compact session rows use `8px` corners and restrained vertical padding.
|
||||
- **Default / Hover / Active:** porcelain at rest, Sunken Surface on hover, and a muted Navigation
|
||||
Active red with a defined border when current. Exactly one top-level navigation control is current.
|
||||
- **Administrative controls:** the admin-only Administration accordion groups Database management,
|
||||
a structural divider, Workspace management, and Pi management in that order. Its trigger exposes
|
||||
- **Administrative controls:** the admin-only Administration accordion groups Database,
|
||||
Memory, Evidence, a structural divider, Workspace, and Pi configuration in that order. Its trigger exposes
|
||||
expanded state and starts collapsed by default, while non-admin users do not receive the accordion
|
||||
or its navigation actions.
|
||||
- **Responsive:** collapse navigation structurally at the application breakpoint. Do not shrink
|
||||
labels into illegibility.
|
||||
labels into illegibility. Below 768px, Memory and Evidence management use the full content
|
||||
width; a Navigation button opens the shared accessible dialog. Selecting another archive
|
||||
page or pressing Escape closes it. Desktop retains the right session sidebar and its My sessions /
|
||||
All sessions tabs. Core retains question/answer, eight phases, reviewer gates and the left log.
|
||||
In embedded mode the portal owns the red header and left sidebar; ThothII must not duplicate them. Size to the
|
||||
actual application container. Narrow session document panels may use the available width.
|
||||
|
||||
### Session review and confirmations
|
||||
|
||||
Session dialogs use the visible application area, including the portal's header
|
||||
and side rail. Artifact and schema-column review can grow to 80rem wide and the
|
||||
available height; short confirmations use up to 40rem and at least 18rem when
|
||||
space permits. Keep a 24px outer margin on desktop and 8px on small or short
|
||||
screens. Long review content scrolls internally; on very short screens the
|
||||
whole dialog can also scroll so every action remains reachable.
|
||||
|
||||
Session forms and review gates repeat their existing primary confirmation above
|
||||
and below the content, sharing selection, validation, pending state and response
|
||||
handlers. Alternate-response inputs follow the same rule. Reserved navigation
|
||||
controls remain below the review. Stop/delete initially focus Cancel; rename
|
||||
initially focuses the name field. Administration dialogs and forms retain their
|
||||
existing layout and actions.
|
||||
|
||||
### Tabs
|
||||
|
||||
- **Shape:** compact label tabs sit on a shared baseline with rounded top corners and a two-pixel
|
||||
lower edge. Inactive labels retain a complete Quiet Border and Porcelain Card surface, so every
|
||||
lower edge, except session-scope tabs which use a uniform one-pixel border, rounded
|
||||
corners and a 4px gap without a shared border or negative bottom margin.
|
||||
Inactive labels retain a Quiet Border and Porcelain Card surface, so every
|
||||
label reads as a tab before interaction; hover feedback reinforces clickability.
|
||||
- **Current:** the selected tab uses the muted Navigation Active red for its fill, text, and defined border.
|
||||
It must expose `aria-selected`, participate in a labelled `tablist`/`tabpanel`, and be the only
|
||||
@@ -319,9 +385,42 @@ default, hover, focus, active, disabled, loading, and error behavior where those
|
||||
|
||||
### Curated Evidence Documents
|
||||
|
||||
Memory and Evidence share the `thot-knowledge-reader` reading contract. Use locally
|
||||
bundled Manrope with normal tracking for prose and labels, and these fixed roles:
|
||||
|
||||
- Card title: 24px, weight 600, line-height 1.3 (`thot-knowledge-title`).
|
||||
- Field/section heading, including Scope and Provenance: 20px, weight 600,
|
||||
line-height 1.4, 8px clearance below (`thot-knowledge-heading`).
|
||||
- All narrative text, including scope, lists and provenance: 16px, weight 400,
|
||||
line-height 1.65. Do not apply compact UI text sizes to these fields.
|
||||
- Authored Markdown subheadings inside a field: 16px, weight 600, line-height 1.5,
|
||||
24px above/8px below. They remain subordinate to the enclosing field heading;
|
||||
their semantic heading levels and original content are preserved.
|
||||
- Technical metadata labels/values: 14px/1.5, with weight 600 for labels.
|
||||
Only code, paths and machine identifiers use the technical monospace family at
|
||||
14px/1.65, identical for inline and fenced code (never compound `em` shrinkage).
|
||||
|
||||
Separate reading sections by 24px; keep the first Markdown block flush with its
|
||||
field heading's 8px bottom gap. The same typography applies in light/dark and at
|
||||
all responsive widths. Controls and archive indexes retain their compact UI roles.
|
||||
|
||||
Memory and Evidence detail readers use the entire available content width, without
|
||||
the ordinary 72–75ch prose cap. This is the owner's explicit reading-layout choice.
|
||||
Long unstructured paragraphs are split for display at existing sentence/semicolon
|
||||
boundaries outside inline code and links; authored Markdown structure and stored
|
||||
content are unchanged. Paragraph spacing is 1.25em. Scope and provenance share the
|
||||
available width; provenance excerpts render Markdown rather than literal markers.
|
||||
Copy actions use the two-overlapping-sheets icon, an accessible name/tooltip and
|
||||
live success/failure feedback instead of a visible Copy label.
|
||||
|
||||
Memory has four explicitly FAKE formatting examples, one per family, in a separate
|
||||
expandable section. They reuse the real detail reader but never enter persistence,
|
||||
indexing, link search or model recall, and expose no edit/delete/save actions.
|
||||
|
||||
Curated evidence follows a fixed reading order: title, compact type and purpose summary, scope,
|
||||
typed content, supporting excerpts, review items, then collapsed technical provenance. Machine
|
||||
metadata stays in invisible comments so GitHub Preview shows only the reviewable document.
|
||||
typed content, supporting excerpts, review items, then technical provenance. Curated v4 files
|
||||
use short, visible YAML frontmatter for identity and classification. The Markdown title and
|
||||
body are authoritative; hidden payload comments are a legacy format converted on consolidation.
|
||||
|
||||
`applies_to` is rendered as “Ambito di applicazione” with separate bullet lists for concepts,
|
||||
tables, and columns. Enum values also use lists. Tables are forbidden for metadata, scope, or any
|
||||
@@ -329,7 +428,9 @@ one-dimensional collection; reserve tables for genuinely two-dimensional dataset
|
||||
identifiers use inline code. SQL uses fenced code. Supporting excerpts use blockquotes.
|
||||
|
||||
**The Review Surface Rule.** The visible Markdown must be readable without understanding the
|
||||
machine contract. Technical metadata belongs in progressive disclosure, not above the title.
|
||||
machine contract. In Administration, explain current and original provenance separately and
|
||||
keep file-editing templates and Git instructions in progressive disclosure. Show actual host
|
||||
paths with copy controls, never browser file links to container-only locations.
|
||||
|
||||
## Do's and Don'ts
|
||||
|
||||
@@ -341,7 +442,8 @@ machine contract. Technical metadata belongs in progressive disclosure, not abov
|
||||
- **Do** preserve information density with headings, rhythm, and progressive disclosure.
|
||||
- **Do** keep keyboard focus explicit and pair color with text, shape, icon, or position.
|
||||
- **Do** respect `prefers-reduced-motion` while preserving immediate non-kinetic feedback.
|
||||
- **Do** use English for interface chrome and the workspace language for persisted document content.
|
||||
- **Do** use the selected interface language (English by default) for chrome and preserve the
|
||||
workspace language for persisted domain content. Session interaction language remains pinned.
|
||||
- **Do** render curated metadata and scope as Markdown prose or lists, never as a frontmatter table.
|
||||
- **Do** break long curated rules into paragraphs, labelled subsections, and lists at existing
|
||||
punctuation boundaries while preserving the exact canonical text for machines.
|
||||
|
||||
+91
-291
@@ -1,309 +1,109 @@
|
||||
# ThothII — Project State
|
||||
# Project state
|
||||
|
||||
Last updated: 2026-09-06.
|
||||
Updated: 2026-09-28. This is a current snapshot, not a release diary. Stable commands
|
||||
and invariants are in [AGENTS.md](AGENTS.md); prior snapshots remain in Git.
|
||||
|
||||
This file is the short operational snapshot. Stable commands and the architecture mental model
|
||||
live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`,
|
||||
`docs/contracts/`, `docs/adr/`, and `docs/evidence.md`. Superseded plans and reports are
|
||||
available from Git history rather than duplicated in the working tree.
|
||||
## Current contracts
|
||||
|
||||
The guarded server migration from a legacy checkout to the schema-v2 installation, Gitea source,
|
||||
Authentik, internal catalog/embedding services, and the PSD workspace repository is documented in
|
||||
`docs/operations/server-upgrade-gitea-workspace-v2.md`. Treat its operator gates and rollback
|
||||
requirements as mandatory; do not replace the running server stack in place.
|
||||
- Document-first installation ticket #43 provides offline `tht workspace prepare`
|
||||
and `tht workspace validate` through a native two-executable bundle. Build and
|
||||
test instructions are in [the host CLI guide](tools/tht/README.md). Local validation
|
||||
reuses runtime workspace/catalog parsers and explicitly defers runtime Evidence,
|
||||
database binding and readiness checks. Ticket #44 adds `tht installation prepare`,
|
||||
explicit `installation credentials`, and `installation validate --workspaces PATH`
|
||||
for protected application documents, canonical model settings and schema-v1
|
||||
database bootstrap inputs. These commands do not start services or import Catalog
|
||||
bindings. Ticket #45 adds host `installation preflight` and `installation plan`
|
||||
with release/image checks, canonical external diagnostics and private input seals;
|
||||
see [the preflight reference](docs/install/installation-preflight.md).
|
||||
The remainder of the installation tickets,
|
||||
Docker Hub publication and example databases remain pending.
|
||||
|
||||
## Current product shape
|
||||
- React supports full/embedded rendering independently of local/OIDC/upstream auth,
|
||||
with EN/IT UI and immutable session interaction language. See
|
||||
[application shell](docs/architecture/application-shell.md) and
|
||||
[localization](docs/operations/shell-and-localization.md).
|
||||
- PostgreSQL Metadata Catalog owns database identity, binding, schema, descriptions,
|
||||
sensitivity and relationships for all core consumers. Workspace schema v4 contains
|
||||
identity and optional Evidence only. Installation schema v2 is the authored model
|
||||
catalog source. See [overview](docs/architecture/overview.md) and
|
||||
[model configuration](docs/general/pi-configuration.md).
|
||||
- The harness owns workflow persistence; chat is not the durable session record.
|
||||
Memory uses PostgreSQL authority and derived Qdrant dense/BM25 search. Editable
|
||||
Evidence has local archive authority and manual consolidation. File save, search
|
||||
activation and Git publication have distinct outcomes. See
|
||||
[Memory](docs/gestione-memory.md), [Evidence](docs/contracts/curated-evidence-v4.md)
|
||||
and [consolidated release evidence](docs/reports/knowledge-archives-release.md).
|
||||
- Reference preprocessing must not clear Memory. Use the installation-scoped
|
||||
`tht --installation /absolute/path/thothii-installation.yaml workspace preprocess run`
|
||||
and its [contract](docs/contracts/workspace-preprocessing-cli.md).
|
||||
- DWH sessions are read-only. SSH tunnels support database-management diagnostics
|
||||
and metadata synchronization, not NL→SQL session creation; use direct or REST
|
||||
transport for sessions.
|
||||
|
||||
ThothII is a human-in-the-loop datamart builder with three independently built layers:
|
||||
## Installation and workspace boundaries
|
||||
|
||||
```text
|
||||
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
|
||||
```
|
||||
Fresh standalone installations follow the manual terminal procedures in
|
||||
[Italian](docs/install/standalone-manual-it.md) or
|
||||
[English](docs/install/standalone-manual-en.md), without an installer or launcher.
|
||||
The [Compose reference](docs/operations/compose-reference.md) is for maintainers,
|
||||
not another quick start. Catalog and Memory migrations are explicit.
|
||||
|
||||
The harness owns the deterministic eight-phase NL→SQL workflow and all session persistence.
|
||||
The backend remains a process/RPC/SSE bridge for sessions and now also owns an isolated PostgreSQL
|
||||
metadata catalog for administrative database configuration. The frontend renders the review gates
|
||||
and keeps the live transcript in memory. See
|
||||
`docs/architecture/components.md` for the detailed component and data-flow map.
|
||||
Descriptors, authentication, provider credentials, certificates and runtime bindings
|
||||
stay in protected installation-local paths. Do not copy secrets into examples or
|
||||
workspace Git. Legacy runtime snapshots may use absolute paths and protected
|
||||
`harness/.env`; do not silently relocate them.
|
||||
|
||||
## Evidence restructuring — accepted
|
||||
PSD authoring is separate at `/Users/mp/projects/tht-workspace-psd`. Its GitHub
|
||||
repository was copied to private Gitea
|
||||
[workspace_psd](https://git.tylconsulting.it/mptyl/workspace_psd), preserving both
|
||||
branches. It is a copy, not automatic synchronization. Running installations were
|
||||
not repointed to a different workspace remote.
|
||||
|
||||
The evidence restructuring and PSD migration completed real acceptance on 2026-08-25.
|
||||
## Recorded deployments and server authority
|
||||
|
||||
- The curated PSD revision contains 35 approved Evidence units and 60 review items.
|
||||
- The PSD authoring repository publishes all 35 units using Curated unit schema v3. Its table-free
|
||||
presentation uses hidden canonical metadata, wrapping Markdown scope lists, list-based enum
|
||||
values, and collapsed technical provenance. Long domain rules now have a deterministic
|
||||
human-readable presentation while retaining their exact canonical text for vector ingestion.
|
||||
`tht evidence migrate <workspace-root>` performs the deterministic v1/v2 upgrade and older-v3
|
||||
presentation rewrite without model calls. The structured-rule PSD rewrite is currently local and
|
||||
pending commit/publication.
|
||||
- The accepted snapshot is
|
||||
`psd-clinical-675990d90eae51da6f2bd51b1ae2609f245772ef-snapshot`.
|
||||
- The active generation is `gen:f968b3bd7a553dbfef3cf47093698f2bc7f95f11`.
|
||||
- Retrieval acceptance reached 20/20 Hit@10.
|
||||
- A real session, `20301df7-cad7-403d-a4c1-9f35c9d07b66`, completed F1–F8 with five
|
||||
receipts, three CTEs, and a final result of 78 patients.
|
||||
- The durable acceptance record is
|
||||
`docs/testing/evidence/evidence-restructuring-psd-acceptance-2026-08-25.md`.
|
||||
Use the ordered [server handoff](docs/operations/server-codex-handoff.md) for
|
||||
coordinated ThothII/Omics upgrades. Omics integration uses GitHub
|
||||
`Dallavilla-Tiziano/omics_portal`, with no Gitea relay prerequisite. Omics uses
|
||||
embedded/upstream identity, not another ThothII OIDC login. Read
|
||||
[upstream authentication](docs/install/authentication-upstream.md) before changes.
|
||||
|
||||
The canonical authoring, validation, publication, materialization, and preprocessing flow is
|
||||
documented in `docs/evidence.md`. The governing contracts are
|
||||
`docs/contracts/workspace-evidence-v3.md` and
|
||||
`docs/contracts/workspace-preprocessing-cli.md`.
|
||||
The incremental server procedure for the `260906-preprocessing-complete` release is
|
||||
`docs/operations/server-handoff-260906-preprocessing-complete.md`.
|
||||
Last recorded application deliveries (not a fresh runtime attestation):
|
||||
|
||||
## Workspace preprocessing and configuration
|
||||
- [Session dialogs](docs/reports/2026-09-14-session-dialogs-release.md):
|
||||
`b1723c34-session-dialogs-20260914`, frontend-only.
|
||||
- [Session layout/Memory fix](docs/reports/2026-09-14-session-layout-memory-fix.md):
|
||||
`49333a2d-session-memory-fix`, core/frontend.
|
||||
|
||||
The native host CLI `tht` is the operator surface. Workspace preprocessing runs through:
|
||||
Keep those reports and rollback instructions while operator gates remain open.
|
||||
A later deployment does not prove every earlier acceptance item passed.
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/thothii-installation.yaml \
|
||||
workspace preprocess run --workspace <workspace-id>
|
||||
```
|
||||
## Remaining acceptance and design gates
|
||||
|
||||
This complete one-shot command uses the profile-gated `workspace-maintenance` service. Partial DWH,
|
||||
schema, Evidence, and vector mutation commands are retired.
|
||||
- Fresh-machine Mac, Windows/WSL2 and Linux installation acceptance, including real
|
||||
DWH/model endpoints, remains a separate operator exercise.
|
||||
- Real IdP/portal login, logout, embedded interaction and PSD semantic acceptance
|
||||
follow the [manual matrix](docs/testing/authentication-manual-acceptance.md) and
|
||||
delivery reports; synthetic tests do not close them.
|
||||
- [Security hardening](docs/plans/2026-09-08-security-hardening-prd.md) is a draft.
|
||||
Revalidate SEC01–12 and obtain design approval before implementation or real
|
||||
server/IdP/DWH mutation.
|
||||
- Optional NER remains opt-in; labeled Italian quality, benchmark and licensing
|
||||
acceptance are not implied by document cleanup.
|
||||
- Semantic aliases, value descriptions, synonyms/concepts, dialect and multi-schema
|
||||
extensions remain explicit design work. Current sensitivity delivery follows the
|
||||
Catalog contract; additional policies require their own acceptance.
|
||||
- Legacy database UI fallback (`?db-ui=legacy`, dev/staging) and prototype removal
|
||||
remain subject to owner acceptance.
|
||||
|
||||
The right Administration sidebar invokes that same operation for the selected workspace. It shows
|
||||
only current readiness or the latest bounded failure diagnostic; there is no preprocessing history.
|
||||
Known non-ready state disables **New session**, while backend admission remains authoritative.
|
||||
The same control exposes an inline-confirmed **Clear** action to remove replaceable reference
|
||||
vectors, LSH, corpus, and checkpoints while preserving the separate Memory collection. The host CLI
|
||||
equivalent is `workspace preprocess clear`.
|
||||
## Documentation maintenance
|
||||
|
||||
Each workspace now uses `<workspace>-reference` for Schema, relationships, and Evidence and
|
||||
`<workspace>-memory` for `memory` and `solved_question`. Clear and preprocessing own only the former.
|
||||
LSH ownership additionally binds the Catalog database ID and Metadata Content Revision, so derived
|
||||
values cannot be reused across database identities or Catalog revisions.
|
||||
MkDocs publishes only 20 product/operator pages and five approved assets.
|
||||
Architecture, contracts, ADRs, plans, research, tests and release evidence are
|
||||
excluded from HTML and search. The repository itself is public: editorial exclusion
|
||||
is not confidentiality.
|
||||
|
||||
Workspace descriptors use schema v4 and contain only workspace identity and optional Evidence.
|
||||
PostgreSQL Metadata Catalog owns database identity, binding, schema, descriptions, sensitivity, and
|
||||
relationships; model, provider, embedding, and vector-store configuration is installation-owned. For
|
||||
PSD, workspace content and runtime roots point to the
|
||||
separate uncommitted repository `/Users/mp/projects/tht-workspace-psd`. Secrets remain outside
|
||||
Git and are supplied only through installation-local protected files.
|
||||
|
||||
## Installation Model Catalog
|
||||
|
||||
`thothii-installation.yaml` schema version 2 is the only operator-authored source for session,
|
||||
metadata-generation, and embedding models. The host `tht` lifecycle validates `modelCatalog` and
|
||||
regenerates the backend catalog, Pi `models.json`/`settings.json`, and Compose override under the
|
||||
installation-local `generated/` directory. Those projections are replaceable runtime adapters:
|
||||
they are not edited, backed up, or treated as configuration.
|
||||
|
||||
Session and metadata defaults use canonical `provider/model` IDs. Provider authentication declares
|
||||
one explicit mode (`secret_env`, `pi_auth`, or `none`); `secret_env` names a protected bundle key.
|
||||
The backend settings store now owns only the selected workspace and thinking level. Existing v1
|
||||
installations use the explicit catalog migration command; schema-v3 workspace descriptors are
|
||||
converted deterministically in their curator-owned repository before commit. Strict runtime loading
|
||||
does not silently infer or merge legacy sources. ADR 0013 and
|
||||
`docs/plans/2026-09-02-installation-model-catalog.md` record the decision and implementation.
|
||||
|
||||
## Database management
|
||||
|
||||
The database, table, and authoritative physical-schema catalog slices are implemented. Database
|
||||
Management now opens the Fleet Ledger presentation by default inside `AppShell`, lists every YAML
|
||||
workspace, creates at most one PostgreSQL database configuration per workspace, edits direct
|
||||
PostgreSQL, REST API, or SSH-tunnel installation bindings, replaces write-only encrypted secrets,
|
||||
and tests supported connector bindings. The surface keeps one responsive AG Grid visible at a time:
|
||||
databases lead to tables, tables lead to columns, and relationships are a sibling database view.
|
||||
Parent navigation remains explicit through the breadcrumb and emphasized back control.
|
||||
|
||||
Selection-scoped operations use one action selector plus an explicit **Run** control; ineligible
|
||||
actions remain visible with their disabled reason, while row-scoped actions stay in the pinned final
|
||||
column. The KPI strip reads installation-wide or selected-database aggregates from
|
||||
`GET /catalog/metrics`. Database configuration, metadata editors, synchronization history,
|
||||
description history, and sensitive-field review/history use the production APIs in right-side
|
||||
drawers rather than prototype fixtures; closing a history drawer does not stop its background run.
|
||||
|
||||
Sensitive-field review is now driven by the versioned local `sensitivity-v4` policy, not by a
|
||||
catalog model. The backend reads selected source tables through read-only, database-specific
|
||||
adapters and makes every `sensitive | non_sensitive` draft decision in the TypeScript
|
||||
`SensitivityClassifier`. A single validated match protects the column. Tables up to 1,000 rows are
|
||||
fully scanned; larger tables use breadth-first 300, 1,000, and text-only 3,000-value targets, with a
|
||||
five-second limit per source query and no global request deadline. Source failures fail the run
|
||||
instead of yielding `unknown`; coverage remains visible separately from the proposal. Draft
|
||||
assessments remain transient until an administrator explicitly saves them. Optional GLiNER2
|
||||
evidence is CPU-only, offline, opt-in, and never replaces the deterministic decision point; see
|
||||
`docs/operations/sensitivity-analysis.md`. The earlier v1 PSD shadow comparison kept NER disabled by
|
||||
default; see `docs/reports/2026-09-02-psd-sensitivity-shadow.md`. The v2 comparison completed all
|
||||
2,275 columns: CPU NER added 18 sensitive proposals and increased warm runtime from 50.1 to 61.3
|
||||
seconds; see `docs/reports/2026-09-03-psd-progressive-sensitivity-shadow.md`.
|
||||
Version 4 excludes declared `bigint` primary-key columns and conventionally named `pk bigint`
|
||||
columns before source inspection, reporting both as non-informative structural identifiers while
|
||||
distinguishing declared constraints from inferred roles.
|
||||
|
||||
Physical membership, source
|
||||
comments, column types/default/nullability/PK positions, and constraint-level ordered FK pairs are
|
||||
projections of the external schema. They cannot be created, renamed, or structurally edited by
|
||||
hand, but administrators can explicitly clear catalog tables, columns, or relationships without
|
||||
touching the source database, binding, configuration, or secrets. Table deletion cascades through
|
||||
columns and relationships; table-scoped relationship cleanup includes incoming and outgoing
|
||||
relationships. Curated and generated descriptions are editable; generated descriptions start null
|
||||
and Database Management can generate or consolidate them for selected tables, selected columns,
|
||||
all targets, or only targets whose Generated Description is missing.
|
||||
|
||||
Relationship Management is now reachable directly from each configured Fleet database. One
|
||||
Relationship Map shows read-only Physical Relationships together with Generated and Manual Logical
|
||||
Relationships, with Active, Excluded, and All filters. Administrators can add a single-column
|
||||
relationship, run deterministic name/PK/type inference, exclude or restore a logical relationship,
|
||||
or delete it permanently. Exclusion retains a tombstone that a rebuild cannot reactivate; permanent
|
||||
deletion allows a later rebuild to infer the same endpoints again. Inference uses no LLM, embedding,
|
||||
or source values. It supports normalized table-qualified names, unique non-generic PK names,
|
||||
composite-PK source columns, and the `*time_key -> dim_time.<single PK>` warehouse convention while
|
||||
ignoring bare generic names. Explicit table/column metadata cleanup remains a destructive boundary: it removes
|
||||
the attached logical relationships and exclusions and requires a full schema synchronization before
|
||||
inference or runtime publication can continue.
|
||||
|
||||
The previous Database Management renderer remains a temporary comparison fallback for development
|
||||
and staging only: `?db-ui=legacy` is honored in Vite development or when
|
||||
`VITE_DB_MANAGEMENT_LEGACY=true`; it is not a production presentation. The standalone Fleet Ledger
|
||||
prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
|
||||
after which both migration aids can be removed.
|
||||
|
||||
Schema refresh is one durable asynchronous engine with database-table, selected-table-column,
|
||||
relationship, and full-database actions. Database-level menus expose only the table, relationship,
|
||||
and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections
|
||||
also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
|
||||
one-active-operation-per-database exclusion with synchronization. Runs have leases and
|
||||
restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
|
||||
apply, retained history, and a live SSE log with polling fallback. Null metadata renders blank
|
||||
rather than as a placeholder.
|
||||
|
||||
Direct PostgreSQL and strict known-host-verified OpenSSH use `pg_catalog`. REST bindings use the
|
||||
typed full-snapshot `POST /rpc/schema_snapshot` contract when available. Servers such as the
|
||||
current PSD endpoint that exposes only `POST /rpc/run_query` use one catalog-owned read-only query
|
||||
to return the exact same strict v1 snapshot in a single round trip. Both paths remain fail-closed:
|
||||
an absent capability, query error, partial result, or invalid snapshot applies no catalog changes.
|
||||
SSH is not yet enabled for NL→SQL session runtime.
|
||||
|
||||
The catalog runs in the internal `catalog-db` PostgreSQL service. Kysely migrations are an explicit
|
||||
one-shot `catalog-migrate` operation; `scripts/run-stack.sh` runs it before local startup. Runtime
|
||||
sessions consume an immutable Catalog JSON snapshot tied to the runtime-config lease. It contains
|
||||
the tables, columns, effective descriptions, sensitivity flags, and active relationships used by
|
||||
the harness; PostgreSQL is the exclusive runtime authority for database metadata. Authored
|
||||
workspace YAML remains limited to workspace identity and optional Evidence configuration. The
|
||||
accepted design is recorded in ADR 0016 and the contracts under `docs/contracts/`.
|
||||
|
||||
Semantic aliases, value descriptions, synonyms, and concepts remain deferred to their dedicated
|
||||
slices.
|
||||
|
||||
AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to
|
||||
`false`, including for newly synchronized columns. An administrator may request a local sensitivity
|
||||
analysis for one selected database, selected tables, or selected columns. One deterministic
|
||||
TypeScript classifier combines metadata, bounded source-content rules, and optional CPU-only NER;
|
||||
no generative model decides the result. Its `sensitive` or `non_sensitive` assessments remain an
|
||||
unsaved draft until the human reviews and saves any chosen flag changes, including a downgrade to
|
||||
non-sensitive. Coverage is reported separately; interrupted history may count unprocessed columns.
|
||||
Each started analysis records a separate Sensitivity Analysis Run with aggregate counters and safe
|
||||
ordered events. The progress drawer opens before the synchronous request completes, polls the run,
|
||||
and displays sanitized source-scan and local-NER phase/batch activity while classification is in
|
||||
progress. This operational history never stores per-column assessments, source values,
|
||||
matched spans, prompts, or free-form diagnostics. Saving a sensitive decision persists a sanitized
|
||||
Sensitivity Reason as column Catalog Metadata alongside the human-owned flag; clearing the flag
|
||||
clears that reason. Reloading still discards an unsaved review draft.
|
||||
For unprotected columns, up to five source rows and five representative non-null values may be sent
|
||||
transiently to the configured model provider. Protected columns are omitted from source reads and
|
||||
replaced in the prompt by deterministic plausible values derived only from their metadata. Existing
|
||||
descriptions are not regenerated when a flag changes.
|
||||
|
||||
The accepted AI-description design is recorded in
|
||||
`docs/plans/2026-08-28-ai-catalog-description-generation.md`, with the formal specification in the
|
||||
adjacent `-spec.md` document and Gitea issue #4. Gitea issues #5–#11 deliver the implementation.
|
||||
The runtime deliberately keeps ThothAI's simple operating model: one installation-wide sequential
|
||||
run owned by the backend, one short-lived Python/LiteLLM completion helper per request, and
|
||||
persistence limited to the run, its safe ordered text events, and each Generated Description as
|
||||
soon as it succeeds. The helper performs at most one provider retry and never falls back to another
|
||||
model. Stop terminates the current helper and retains prior results; three consecutive exhausted
|
||||
technical batches fail the run. Startup marks stale queued/running work interrupted, and Unlock is
|
||||
available only when no local start, worker, or helper is live. Runs remain inspectable through a
|
||||
live SSE log with ordered polling fallback; there is no automatic resume or user-facing generation
|
||||
CLI. ADRs 0009–0010 record the runtime and source-sampling decisions.
|
||||
|
||||
The Installation Model Catalog accepts the protected `DEEPSEEK_API_KEY` and `ZAI_API_KEY`
|
||||
references for metadata-generation providers.
|
||||
It also accepts a model with no secret reference only when its OpenAI-compatible endpoint is
|
||||
explicit; this covers the VPN-only AritmoLab Qwen 3.6 server without creating a fake operator
|
||||
credential. The Python client supplies only its fixed non-secret compatibility placeholder.
|
||||
The AritmoLab entry also sets `disableThinking: true`, mapped to the endpoint's chat-template flag,
|
||||
because its default reasoning prose would violate the worker's exact JSON response contract.
|
||||
|
||||
Logical relationship integration with core schema-linking is complete: session creation and resume
|
||||
materialize the active physical/generated/manual map, retrieval-pack generation and Pi receive the
|
||||
same runtime config, and snapshot validation fails closed on a declared missing, invalid, or orphaned
|
||||
endpoint. Broader publication of other Catalog metadata to schema-linking remains a separate future
|
||||
slice.
|
||||
|
||||
**Deferred follow-up — Sensitive Data Policy in schema-linking.** The policy is first delivered
|
||||
and tested in catalog description generation. Its enforcement for core schema-linking remains
|
||||
out of scope until the current tickets are closed and the owner has completed the acceptance test.
|
||||
At that gate, resume the design: `tht` must receive a read-only projection of the current Sensitive
|
||||
Data Flags and exclude values from columns marked sensitive from every LSH result before it is
|
||||
given to Pi. Do not start this integration before the owner gives final approval after that test.
|
||||
|
||||
## Active deployment work and manual gates
|
||||
|
||||
### PSD server deployment program
|
||||
|
||||
The approved design and executable entry point are:
|
||||
|
||||
- `docs/plans/2026-08-20-psd-server-deployment-program-design.md`
|
||||
- `docs/plans/2026-08-20-psd-server-deployment-program.md`
|
||||
- `docs/plans/2026-08-20-psd-server-survey.md`
|
||||
- `docs/plans/2026-08-20-psd-server-project-a-standalone.md`
|
||||
- `docs/plans/2026-08-20-psd-server-project-b-authentik.md`
|
||||
|
||||
Last recorded state:
|
||||
|
||||
- survey: `SURVEY_NO_GO`;
|
||||
- Project A: `BLOCKED_BY_SURVEY_AND_MUTATION_GATE`;
|
||||
- Project B: `BLOCKED_BY_PROJECT_A_AND_PRE_B_GATE`.
|
||||
|
||||
The deployment is a clean replacement: legacy sessions, indexes, and application configuration
|
||||
are not migration inputs. The existing stack remains intact until its documented mutation and
|
||||
rollback gates are explicitly approved. Shared Omics/LocalLLM networks, ETL Evidence, DWH,
|
||||
`dwh-auth`, Supabase, Authentik, Superset, and Aritmolab are outside cleanup scope.
|
||||
|
||||
Human acceptance guides and sanitized report templates live under `docs/testing/` and
|
||||
`docs/testing/evidence/`. The remediation checklist is
|
||||
`docs/operations/psd-server-survey-remediation-checklist.md`.
|
||||
|
||||
### Authentication
|
||||
|
||||
The local/OIDC authentication remediation passed its automated review on 2026-08-18. Release and
|
||||
PSD mutation gates remain governed by:
|
||||
|
||||
- `docs/architecture/authentication.md`;
|
||||
- `docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md`;
|
||||
- `docs/operations/psd-dwh-auth-rollout.md`;
|
||||
- `docs/testing/authentication-manual-acceptance.md`.
|
||||
|
||||
Do not infer authorization for server, Nginx, Authentik, database, credential, or cutover changes
|
||||
from an automated PASS.
|
||||
|
||||
## Verification status
|
||||
|
||||
- The P1.1 workspace-directory registry and P2–P6 preprocessing workstreams are implemented and
|
||||
have automated coverage.
|
||||
- Evidence restructuring has a real PSD acceptance PASS as recorded above.
|
||||
- AI Description Generation has automated coverage across installation setup, model selection,
|
||||
generation/consolidation scopes, bounded sampling, cancellation/recovery, history, SSE/polling,
|
||||
and the LiteLLM helper boundary.
|
||||
- L2 tests requiring real providers or remote databases remain opt-in.
|
||||
- Server deployment, release, and owner-operated acceptance steps remain pending wherever the
|
||||
referenced runbooks require explicit approval.
|
||||
|
||||
Run the layer-specific checks documented in `AGENTS.md`. For release-sensitive changes, also run
|
||||
the repository contract scripts in `scripts/` and build the MkDocs site.
|
||||
|
||||
## Operational invariants
|
||||
|
||||
- `tht`'s `-c`/`--config` option follows the subcommand; it is not a global option.
|
||||
- `--json` commands write pristine JSON to stdout.
|
||||
- Persisted phase documents and the decision ledger are the source of session truth; chat is not.
|
||||
- UI chrome is English; workspace document content retains the workspace language.
|
||||
- The backend refuses resume for finalized or archived sessions.
|
||||
- A resume must send `/riprendi-sessione <id>`; a new session must send `/nuova-domanda`.
|
||||
- DWH access is read-only.
|
||||
The [cleanup record](docs/maintenance/2026-09-15-documentation-cleanup.md) records
|
||||
retired sources and retained gates. Main contains source; Actions generates the
|
||||
`pages` branch. The live site requires the separate explicit deployment described
|
||||
in [public manual publication](docs/operations/public-docs-publication.md).
|
||||
|
||||
@@ -4,72 +4,52 @@ ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fasti
|
||||
core. The portable deployment runs two application services plus the installation-local metadata
|
||||
catalog; DWH and LLM services remain external. Semantic services are bundled in Compose.
|
||||
|
||||
Authentication is configured through the single host CLI tht: see the [local authentication guide](docs/install/authentication-local.md),
|
||||
[generic OIDC guide](docs/install/authentication-oidc.md), and [manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
|
||||
The same frontend supports **full** (its own header) and **embedded** (inside a
|
||||
portal). This choice is independent of authentication: the Mac uses full/local,
|
||||
Omics uses embedded/upstream with its existing login, and a standalone server
|
||||
can use full/OIDC. See [rendering architecture](docs/architecture/application-shell.md)
|
||||
and [configuration, Omics delivery and deploy](docs/operations/shell-and-localization.md).
|
||||
|
||||
## Docker Compose: local startup
|
||||
For the current server upgrade with Omics Portal, follow the ordered
|
||||
[Codex server handoff](docs/operations/server-codex-handoff.md), including source
|
||||
integration, embedded/upstream configuration, coordinated rollout and rollback.
|
||||
|
||||
Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`,
|
||||
`catalog-db`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external,
|
||||
configurable endpoints—even when they are co-located with ThothII.
|
||||
Local/OIDC authentication is configured through the host CLI `tht`; portal
|
||||
authentication is established by the trusted server proxy. See the
|
||||
[local guide](docs/install/authentication-local.md),
|
||||
[OIDC guide](docs/install/authentication-oidc.md),
|
||||
[upstream integration](docs/install/authentication-upstream.md), and
|
||||
[manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
|
||||
|
||||
From a fresh clone, run these commands from the repository root:
|
||||
For the clone-based manual standalone installation test on macOS, Windows, and Linux, use the
|
||||
[Italian procedure](docs/install/standalone-manual-it.md) or the
|
||||
[English procedure](docs/install/standalone-manual-en.md).
|
||||
|
||||
```sh
|
||||
cp deploy/env/local.env.example deploy/env/local.env
|
||||
# Copy docs/install/examples/thothii-installation.local.yaml to a protected operator path,
|
||||
# replace its placeholders, chmod it 600, and set that exact THT_INSTALLATION_CONFIG_SOURCE.
|
||||
# Edit deploy/env/local.env, including PI_AUTH_FILE, THT_SECRETS_FILE, and external endpoints.
|
||||
./scripts/run-stack.sh
|
||||
```
|
||||
The [public manual](https://git.tylconsulting.it/thothii-docs/) covers the product,
|
||||
installation, use and administration. Developer architecture, contracts, ADRs, tests,
|
||||
plans and release records remain in this repository but are excluded from MkDocs
|
||||
pages and search. This is an editorial boundary, not an access restriction on the
|
||||
public repository. See the [documentation cleanup review](docs/maintenance/2026-09-15-documentation-cleanup.md)
|
||||
for the executed consolidation and the inventory of historical sources retained in Git.
|
||||
|
||||
The launcher builds the core, starts `catalog-db`, runs the explicit one-shot Kysely migrations,
|
||||
then runs the base+local stack in the foreground. Migrations never run implicitly in backend
|
||||
startup. The core image contains its Pi runtime; no host `pi` executable is used. For a server
|
||||
installation, build the image, start the catalog, and run the same migration service before the
|
||||
application rollout:
|
||||
## Docker Compose and installation
|
||||
|
||||
```sh
|
||||
cp deploy/env/server.env.example deploy/env/server.env
|
||||
# Prepare a mode-600 thothii-installation.yaml from the server example and set its exact
|
||||
# path as THT_INSTALLATION_CONFIG_SOURCE. Edit all remaining storage/secret/endpoint paths.
|
||||
sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example build core
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example up -d catalog-db
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example run --rm catalog-migrate
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example up --build -d
|
||||
```
|
||||
For a fresh installation, follow the guided terminal procedure in
|
||||
[Italian](docs/install/standalone-manual-it.md) or
|
||||
[English](docs/install/standalone-manual-en.md). The single
|
||||
`tht setup --complete` command validates protected files, builds the images, runs
|
||||
Catalog migration, starts the stack and imports the configured workspace repository.
|
||||
There is no graphical installer or native launcher.
|
||||
|
||||
The initializer is required for an empty or restored server Pi-state bind. It atomically creates
|
||||
the three regular targets hidden below the writable parent bind; protected Pi auth and tracked
|
||||
model/settings sources remain separate read-only mounts. See the server manual before substituting
|
||||
a root other than `/srv/thothii/pi-state`.
|
||||
The mandatory stack includes frontend, core, PostgreSQL catalog, Qdrant, Ollama
|
||||
and the embedding initializer. DWH and LLM endpoints remain external dependencies.
|
||||
Pi is included in the core image. Credentials and certificates belong in protected
|
||||
installation-local files, never in the workspace repository.
|
||||
|
||||
Workspace descriptors come from the Git remote configured by `THT_WORKSPACE_GIT_REMOTE`; their
|
||||
runtime endpoint and secret bindings remain installation-local. Open
|
||||
<http://127.0.0.1:8080> (set `THOTH_HTTP_PORT` in `deploy/env/local.env` to choose another
|
||||
loopback port).
|
||||
|
||||
Credentials and certificates are local protected files. Do not put them in environment examples,
|
||||
workspace YAML, URLs, or Compose interpolation values.
|
||||
|
||||
Application state is split across the named `settings`, `pi-state`, `workspace-registry`,
|
||||
`sessions`, `qdrant-data`, and `embedding-models` volumes. `docker compose down` keeps them.
|
||||
`qdrant-data` is a derived but persistent index store; `embedding-models` is an Ollama model
|
||||
cache for `qwen3-embedding:0.6b` with fixed `1024`-dimension embeddings. Only an explicit destructive command such as `docker compose
|
||||
down --volumes` removes them.
|
||||
|
||||
The frontend depends on the core health check and proxies `/health` and `/api/*` to it. The
|
||||
application health endpoint intentionally checks process readiness only; external dependency
|
||||
diagnostics are exposed by `tht doctor` and do not prevent the UI from starting.
|
||||
For developer topology, overlays and lifecycle details, see the internal
|
||||
[Compose reference](docs/operations/compose-reference.md). Ordinary stop/down keeps
|
||||
persistent data; removing volumes is destructive and is not an upgrade step.
|
||||
Process health is distinct from external dependency checks performed by doctor.
|
||||
|
||||
## Git-backed workspace repository
|
||||
|
||||
|
||||
Generated
+609
@@ -6,11 +6,13 @@
|
||||
"": {
|
||||
"name": "thothii-backend",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.1141.0",
|
||||
"@fastify/cookie": "11.1.2",
|
||||
"@fastify/cors": "^11.2.0",
|
||||
"@fastify/rate-limit": "11.2.0",
|
||||
"@types/pg": "^8.20.3",
|
||||
"fastify": "^5.0.0",
|
||||
"ipaddr.js": "2.4.0",
|
||||
"kysely": "^0.29.5",
|
||||
"libphonenumber-js": "1.13.12",
|
||||
"openid-client": "6.8.5",
|
||||
@@ -23,11 +25,320 @@
|
||||
"@testcontainers/postgresql": "^12.1.0",
|
||||
"@types/node": "24.13.3",
|
||||
"@types/validator": "13.15.10",
|
||||
"bun": "1.4.2",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0",
|
||||
"vitest": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/checksums": {
|
||||
"version": "3.1001.1",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.1.tgz",
|
||||
"integrity": "sha512-x12Q17KYlJAd3nKf8LV5LV0vt8sh8/6YfQLGPtrGnQf/tW4jqxPGq5GPpuVitpQYM3eUR4XB7CbxZf751NMbLw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/client-s3": {
|
||||
"version": "3.1141.0",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1141.0.tgz",
|
||||
"integrity": "sha512-uOVH37xGLenAdJkCPCin/JJG2PgWrFcSsDnQ9+C9Zq8N9Oalo5ol4xmn5fG28iWAlA/b/9boQZgHbMh+UsIhcg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/checksums": "^3.1001.1",
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/credential-provider-node": "^3.972.84",
|
||||
"@aws-sdk/middleware-sdk-s3": "^3.972.77",
|
||||
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/fetch-http-handler": "^5.8.0",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/core": {
|
||||
"version": "3.978.1",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.1.tgz",
|
||||
"integrity": "sha512-LbY9aGsEiznDWmUc30Nwv3aIX/+dbwTx8KfS0yOC3NPYMO+O91e6jkT1azf34FwjOndq8/Q+RcVVZz5xnerwdg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@aws-sdk/xml-builder": "^3.972.41",
|
||||
"@aws/lambda-invoke-store": "^0.3.0",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/signature-v4": "^5.7.3",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"bowser": "^2.11.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-env": {
|
||||
"version": "3.972.72",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.72.tgz",
|
||||
"integrity": "sha512-xTKO/FWJPozTIXbozVnVGoNBhaGba8TBcx+KyUjRVeOlXE+dUc7GTR1cLvu0uTdIdmemzaFbqqCshXeZA1fZew==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-http": {
|
||||
"version": "3.972.74",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.74.tgz",
|
||||
"integrity": "sha512-u91E/hT8f4d1xy0Jl7VG4nVKJ3lxbrZkoBTeSVoJdWBiSEUMwMS/9+e0H/aJVQV//Lt5wuzP+E69v4aRSsNTmw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/fetch-http-handler": "^5.8.0",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-ini": {
|
||||
"version": "3.973.17",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.17.tgz",
|
||||
"integrity": "sha512-ged4KXdBkvIC81bLvNHHuQKdKak/VXhQTR1NWYTTqW0474nlmsxy9O/vlgTIohDDWH3xpBdtVMZRyjb+DnocDA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/credential-provider-env": "^3.972.72",
|
||||
"@aws-sdk/credential-provider-http": "^3.972.74",
|
||||
"@aws-sdk/credential-provider-login": "^3.972.79",
|
||||
"@aws-sdk/credential-provider-process": "^3.972.72",
|
||||
"@aws-sdk/credential-provider-sso": "^3.973.16",
|
||||
"@aws-sdk/credential-provider-web-identity": "^3.972.78",
|
||||
"@aws-sdk/nested-clients": "^3.997.46",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/credential-provider-imds": "^4.5.2",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-login": {
|
||||
"version": "3.972.79",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.79.tgz",
|
||||
"integrity": "sha512-L+Z85anONJd8MaiuraO4wRxATCdEejBZ3K3eymzWI5JPXa9sOS9CkIm72PBKqXKX+Z9p9NGMX5AIMXm0LEflgw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/nested-clients": "^3.997.46",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-node": {
|
||||
"version": "3.972.84",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.84.tgz",
|
||||
"integrity": "sha512-oHt854odINVwzwsh+c5x69j0ajm4DbqqqVJ+O1ECsCIZeMDAbzFpXItaqP7UZstJj/ATdTk/KFSH0LaNAgV+kA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/credential-provider-env": "^3.972.72",
|
||||
"@aws-sdk/credential-provider-http": "^3.972.74",
|
||||
"@aws-sdk/credential-provider-ini": "^3.973.17",
|
||||
"@aws-sdk/credential-provider-process": "^3.972.72",
|
||||
"@aws-sdk/credential-provider-sso": "^3.973.16",
|
||||
"@aws-sdk/credential-provider-web-identity": "^3.972.78",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/credential-provider-imds": "^4.5.2",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-process": {
|
||||
"version": "3.972.72",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.72.tgz",
|
||||
"integrity": "sha512-rLIp2xbMjX/k9/od7APpqq1ZgXXnV0pOL1Th3ZsL8Wu0TRtBsDTVS8iPqcfRFcHakFxPvR04OSTv2ka2qOb/2A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-sso": {
|
||||
"version": "3.973.16",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.16.tgz",
|
||||
"integrity": "sha512-IGihaJfFZYacJJr/odqILCoK7W/mvrZ7cuK7ECn3sAu4vLC6u0V8bS7mCGbdugJ8Aum2tnvqmx0F2MRFp2rn9g==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/nested-clients": "^3.997.46",
|
||||
"@aws-sdk/token-providers": "3.1138.0",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-web-identity": {
|
||||
"version": "3.972.78",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.78.tgz",
|
||||
"integrity": "sha512-/y9WvNtlcPBGLR0qc1a+9J/xtYZfVczvLUOuXaVWylzttH7ewsxwHtjmiJSolNrVSDorIxHGHMU61CbonRkmwA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/nested-clients": "^3.997.46",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/middleware-sdk-s3": {
|
||||
"version": "3.972.77",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.77.tgz",
|
||||
"integrity": "sha512-E7W2UOeUoc+lg3uIfR/dM7ZwusHwhBQrKMnlkRv4EXRR+C0YtV1pg25xC7GdZIhXH+NAMgZPCbE7o5to2cjFiw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/nested-clients": {
|
||||
"version": "3.997.46",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.46.tgz",
|
||||
"integrity": "sha512-oRxtBcka/JGHGs9l9p9IVajGoTP8vTPmoAzdHGy4Qcy9P5vPnDf6nhIeM/COQNY9k/OahImTRaLkHftoXvfcmQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/fetch-http-handler": "^5.8.0",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/signature-v4-multi-region": {
|
||||
"version": "3.996.47",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.47.tgz",
|
||||
"integrity": "sha512-Zk08macMvQTHzQJCLJVkOlviVoqwYMrpXv4lmLN7b7sAbiMoOK7Go0NYdR5UeF+MW8LIbRmwrNy9u/5VvX1U5g==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/signature-v4": "^5.7.3",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/token-providers": {
|
||||
"version": "3.1138.0",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1138.0.tgz",
|
||||
"integrity": "sha512-GpyAr0DD63YOEmYFM6Df+gJuIgC92MMTiBK4FTKfxii5MJ9ge20epR7LyroulscYlG89J+ZB2ivFDPjvfQhzdw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.1",
|
||||
"@aws-sdk/nested-clients": "^3.997.46",
|
||||
"@aws-sdk/types": "^3.974.6",
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/types": {
|
||||
"version": "3.974.6",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz",
|
||||
"integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/xml-builder": {
|
||||
"version": "3.972.41",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz",
|
||||
"integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws/lambda-invoke-store": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
|
||||
"integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@balena/dockerignore": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz",
|
||||
@@ -802,6 +1113,174 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/@oven/bun-darwin-aarch64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-darwin-aarch64/-/bun-darwin-aarch64-1.4.2.tgz",
|
||||
"integrity": "sha512-MXdZkP1featqxZ+/VTXWG1BVjM4OGBehVY2Q88EeUj/7L0UMeCGItmyPYTN+wxvlGJ6F66JEtzsw+GvQWewnag==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-darwin-x64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-darwin-x64/-/bun-darwin-x64-1.4.2.tgz",
|
||||
"integrity": "sha512-gZTxZuLjkUhAWjTETu3tw0WhsEdNkJ64daj60ybhPf835a2yollV3yTkK9JozvzKPx4TRFzLSl8C+U525pxVbw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-freebsd-aarch64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-freebsd-aarch64/-/bun-freebsd-aarch64-1.4.2.tgz",
|
||||
"integrity": "sha512-SMNItMw1Z8QeeQVKnw8jA7xQNkeXdP+OPgin4Wi/QTx/B8RHHLnuZfqmFy7NtVeT2NF0kKYppW4WWd2CCYZjhQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-freebsd-x64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-freebsd-x64/-/bun-freebsd-x64-1.4.2.tgz",
|
||||
"integrity": "sha512-THbPKXhO54N0DpFRKZNDZpQ7dpbX0bWASuARckAUS9wRtFIHsiY+uULXJvxJGo2YD1YewvXQ4G8Fj7XT5oBCiw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-linux-aarch64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64/-/bun-linux-aarch64-1.4.2.tgz",
|
||||
"integrity": "sha512-3BBP9ovJ2RGHFH6Ae1CAtxNtG1+YY6GD6rmYbsUosoAk9+OEl6zeDQ/k4fBkc6dYOJCtWnx8hUxzNzQATSmvYQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-linux-aarch64-android": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64-android/-/bun-linux-aarch64-android-1.4.2.tgz",
|
||||
"integrity": "sha512-3mZKO2rhsNgbAUtAHC1UKUlF2zTxFraDZT/Elv8wzyH0fJL9h+Iv3TgB9lO63w89PRn3eFe+NRA1bhVgikKNPQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-linux-aarch64-musl": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64-musl/-/bun-linux-aarch64-musl-1.4.2.tgz",
|
||||
"integrity": "sha512-+Sm6y+lSiSFBOtXmnekp5Q6n1tUKlyv71FCPWBc61Cgb14T5eBs8SN/nh4MUCOKzONkI3O+as3MGUgikS4aCBQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-linux-x64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-linux-x64/-/bun-linux-x64-1.4.2.tgz",
|
||||
"integrity": "sha512-9/E/UXOTpSo3YsV5g+FhtTd/qTpiWoKuxS12cqtuYA1ssu9fRAoPQnipFgGyck3tWO63iUdxBiygq+kELFawng==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-linux-x64-android": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-linux-x64-android/-/bun-linux-x64-android-1.4.2.tgz",
|
||||
"integrity": "sha512-6HC5tzcC79113n2IHCTJMWv+HsQImv4ZFEK2XpYLxY6HbT8tM4cUM2Zv1bHZBQsS3jv/zYBamDJ1UX7If0d5tw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-linux-x64-musl": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-linux-x64-musl/-/bun-linux-x64-musl-1.4.2.tgz",
|
||||
"integrity": "sha512-vVTKUg1bnPhRP/Hp73jIVoFh2vPFNYEqYX0ERKfZBOQEEHitNAeukZzzuUDZS0SoDCIpuWUGSpd/CDMbjdR+Uw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-windows-aarch64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-windows-aarch64/-/bun-windows-aarch64-1.4.2.tgz",
|
||||
"integrity": "sha512-8EJ1ST7339WJE3poPW5nBgVW/lWf9HBz4W27ZUNhburKmcBLOByPyE6DP9fHD8FQGm5c+ilUN2hX1mrW0jxq9Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@oven/bun-windows-x64": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@oven/bun-windows-x64/-/bun-windows-x64-1.4.2.tgz",
|
||||
"integrity": "sha512-+bN6OuVld/9diT/RLSXSW7JE6CvNE3gL9XsAEjULi1nUsXd6DNO6GuA9jNdNb3r8PdJFnYHr5aypNV1Oj3Rd9g==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@pinojs/redact": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
|
||||
@@ -1235,6 +1714,87 @@
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@smithy/core": {
|
||||
"version": "3.35.0",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.35.0.tgz",
|
||||
"integrity": "sha512-zRMhfkByhT2snNdr1si24vJitU6Cr9ix2MikUfWmkAgp4jrNP0GcKSP5YvwQ+TlI8AZXER5QOGJn3JsVtSD9/A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/credential-provider-imds": {
|
||||
"version": "4.5.2",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz",
|
||||
"integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.33.2",
|
||||
"@smithy/types": "^4.17.2",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/fetch-http-handler": {
|
||||
"version": "5.8.0",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz",
|
||||
"integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.33.3",
|
||||
"@smithy/types": "^4.18.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/node-http-handler": {
|
||||
"version": "4.12.1",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz",
|
||||
"integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.33.3",
|
||||
"@smithy/types": "^4.18.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/signature-v4": {
|
||||
"version": "5.7.4",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz",
|
||||
"integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/types": {
|
||||
"version": "4.19.0",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz",
|
||||
"integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@testcontainers/postgresql": {
|
||||
"version": "12.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@testcontainers/postgresql/-/postgresql-12.1.0.tgz",
|
||||
@@ -1821,6 +2381,12 @@
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/bowser": {
|
||||
"version": "2.14.1",
|
||||
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
|
||||
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
|
||||
@@ -1876,6 +2442,43 @@
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bun": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/bun/-/bun-1.4.2.tgz",
|
||||
"integrity": "sha512-TrSXo6HJfIEaczpb3kjX82I2pL47vK1QUNmHRCUdz9IzaOwa9lzOXSWwu2l18YHE3sNfGRapVLd4nNm+22vVVA==",
|
||||
"cpu": [
|
||||
"arm64",
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"os": [
|
||||
"darwin",
|
||||
"linux",
|
||||
"android",
|
||||
"freebsd",
|
||||
"win32"
|
||||
],
|
||||
"bin": {
|
||||
"bun": "bin/bun.exe",
|
||||
"bunx": "bin/bunx.exe"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oven/bun-darwin-aarch64": "1.4.2",
|
||||
"@oven/bun-darwin-x64": "1.4.2",
|
||||
"@oven/bun-freebsd-aarch64": "1.4.2",
|
||||
"@oven/bun-freebsd-x64": "1.4.2",
|
||||
"@oven/bun-linux-aarch64": "1.4.2",
|
||||
"@oven/bun-linux-aarch64-android": "1.4.2",
|
||||
"@oven/bun-linux-aarch64-musl": "1.4.2",
|
||||
"@oven/bun-linux-x64": "1.4.2",
|
||||
"@oven/bun-linux-x64-android": "1.4.2",
|
||||
"@oven/bun-linux-x64-musl": "1.4.2",
|
||||
"@oven/bun-windows-aarch64": "1.4.2",
|
||||
"@oven/bun-windows-x64": "1.4.2"
|
||||
}
|
||||
},
|
||||
"node_modules/byline": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
|
||||
@@ -4114,6 +4717,12 @@
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/tsx": {
|
||||
"version": "4.22.4",
|
||||
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz",
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build:workspace-tools": "node scripts/build-workspace-tools.mjs",
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"prebuild": "node scripts/clean-dist.mjs",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
@@ -14,11 +15,13 @@
|
||||
"test:schema-v3-verifier": "npm run test:schema-v4-verifier"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.1141.0",
|
||||
"@fastify/cookie": "11.1.2",
|
||||
"@fastify/cors": "^11.2.0",
|
||||
"@fastify/rate-limit": "11.2.0",
|
||||
"@types/pg": "^8.20.3",
|
||||
"fastify": "^5.0.0",
|
||||
"ipaddr.js": "2.4.0",
|
||||
"kysely": "^0.29.5",
|
||||
"libphonenumber-js": "1.13.12",
|
||||
"openid-client": "6.8.5",
|
||||
@@ -31,6 +34,7 @@
|
||||
"@testcontainers/postgresql": "^12.1.0",
|
||||
"@types/node": "24.13.3",
|
||||
"@types/validator": "13.15.10",
|
||||
"bun": "1.4.2",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0",
|
||||
"vitest": "^2.1.0"
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
// Maintainer-only build. The resulting two-binary bundle needs no extra host runtime.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const repository = process.env.THT_BUILD_SOURCE_ROOT ? resolve(process.env.THT_BUILD_SOURCE_ROOT) : resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const backend = join(repository, "backend");
|
||||
const native = `${process.platform === "win32" ? "windows" : process.platform}-${process.arch === "x64" ? "amd64" : process.arch}`;
|
||||
const targets = {
|
||||
"windows-amd64": ["windows", "amd64", "bun-windows-x64"],
|
||||
"darwin-amd64": ["darwin", "amd64", "bun-darwin-x64"],
|
||||
"darwin-arm64": ["darwin", "arm64", "bun-darwin-arm64"],
|
||||
"linux-amd64": ["linux", "amd64", "bun-linux-x64"],
|
||||
"linux-arm64": ["linux", "arm64", "bun-linux-arm64"],
|
||||
};
|
||||
const requested = process.argv.slice(2);
|
||||
const selected = requested.length === 1 && requested[0] === "--all" ? Object.keys(targets) : requested.length ? requested : [native];
|
||||
if (selected.some((target) => !targets[target])) {
|
||||
console.error(`Usage: npm run build:workspace-tools -- [${Object.keys(targets).join("|")}|--all]`);
|
||||
process.exit(2);
|
||||
}
|
||||
function run(command, args, cwd = backend, env = process.env) {
|
||||
const result = spawnSync(command, args, { cwd, env, stdio: "inherit" });
|
||||
if (result.error || result.status !== 0) throw new Error(`Build failed: ${command}`);
|
||||
}
|
||||
const revision = spawnSync("git", ["rev-parse", "HEAD"], { cwd: repository, encoding: "utf8" });
|
||||
if (revision.status !== 0) throw new Error("Cannot read build revision");
|
||||
const commit = revision.stdout.trim();
|
||||
const buildTime = process.env.THT_BUILD_TIME ?? new Date().toISOString();
|
||||
const releaseVersion = process.env.THT_BUILD_VERSION ?? "0.0.0-dev";
|
||||
if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?$/.test(releaseVersion) || !Number.isFinite(Date.parse(buildTime))) throw new Error("Invalid build identity");
|
||||
const module = "github.com/aritmolab/thothii/tools/tht/internal/version";
|
||||
const bunPackage = JSON.parse(readFileSync(join(backend, "node_modules", "bun", "package.json"), "utf8"));
|
||||
const bun = join(backend, "node_modules", "bun", bunPackage.bin.bun);
|
||||
for (const target of selected) {
|
||||
const [os, arch, bunTarget] = targets[target];
|
||||
const output = join(repository, "dist", "workspace-tools", target);
|
||||
mkdirSync(output, { recursive: true });
|
||||
const extension = os === "windows" ? ".exe" : "";
|
||||
const names = [`tht${extension}`, `tht-workspace-documents${extension}`];
|
||||
run(bun, ["build", "src/workspace-documents-cli.ts", "--compile", `--target=${bunTarget}`, "--outfile", join(output, names[1])]);
|
||||
run("go", ["build", "-trimpath", "-ldflags", `-s -w -X ${module}.semanticVersion=${releaseVersion} -X ${module}.commit=${commit} -X ${module}.buildTime=${buildTime}`, "-o", join(output, names[0]), "./cmd/tht"], join(repository, "tools", "tht"), { ...process.env, CGO_ENABLED: "0", GOOS: os, GOARCH: arch });
|
||||
const hashes = names.map((name) => `${createHash("sha256").update(readFileSync(join(output, name))).digest("hex")} ${name}\n`).join("");
|
||||
writeFileSync(join(output, "SHA256SUMS"), hashes);
|
||||
writeFileSync(join(output, "build.json"), JSON.stringify({ version: releaseVersion, commit, buildTime, target, bun: JSON.parse(readFileSync(join(backend, "package.json"), "utf8")).devDependencies.bun }, null, 2) + "\n");
|
||||
console.log(output);
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env node
|
||||
// Maintainer-only producer. Consumers download the resulting native bundle.
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync, existsSync, realpathSync, renameSync, rmSync, statSync, copyFileSync, chmodSync, openSync, closeSync, readdirSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { parse } from "yaml";
|
||||
import { prepareBundle, sha256 } from "./release-bundle.mjs";
|
||||
import { dockerCredentials, dockerHub } from "./release-registry.mjs";
|
||||
import { giteaHosting } from "./release-hosting.mjs";
|
||||
import { publishVerifiedRelease } from "./release-publication.mjs";
|
||||
|
||||
const repositoryRoot = resolve(import.meta.dirname, "../..");
|
||||
export function optionsFromArgs(args) {
|
||||
const values = {};
|
||||
for (let i = 0; i < args.length; i += 2) {
|
||||
if (!['--revision', '--version', '--namespace', '--platforms', '--output', '--repository'].includes(args[i]) || !args[i + 1] || values[args[i]]) throw new Error("Usage: --revision REF --version VERSION --namespace DOCKER_HUB_NAMESPACE --platforms linux/amd64 --output NEW_OR_MATCHING_DIRECTORY [--repository HTTPS_GITEA_REPO]");
|
||||
values[args[i]] = args[i + 1];
|
||||
}
|
||||
if (!values['--revision'] || values['--revision'].startsWith('-') || !/^[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?$/.test(values['--version'] ?? '') || !/^[a-z0-9][a-z0-9_-]{1,38}$/.test(values['--namespace'] ?? '') || !values['--output']) throw new Error("Supply an explicit source revision, semantic release version, Docker Hub namespace and output directory.");
|
||||
const platforms = (values['--platforms'] ?? '').split(',');
|
||||
if (!platforms.length || new Set(platforms).size !== platforms.length || platforms.some((p) => !['linux/amd64', 'linux/arm64'].includes(p))) throw new Error("Select explicit Linux image platforms; begin with linux/amd64 for Windows/WSL2 and Omarchy.");
|
||||
const repository = values['--repository'] ?? 'https://git.tylconsulting.it/mptyl/ThothII';
|
||||
const url = new URL(repository);
|
||||
if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || !/^\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(url.pathname)) throw new Error("Use a credential-free HTTPS Gitea owner/repository URL.");
|
||||
return { revision: values['--revision'], version: values['--version'], namespace: values['--namespace'], platforms: platforms.sort(), output: resolve(values['--output']), repository };
|
||||
}
|
||||
export function commandRunner(logs) {
|
||||
let sequence = 0;
|
||||
return async (command, args, { cwd = repositoryRoot, input = '', env = process.env, quiet = false, timeout = 120_000 } = {}) => {
|
||||
const log = join(logs, `${++sequence}-${basename(command)}.log`);
|
||||
return new Promise((accept, reject) => {
|
||||
if (process.platform === 'win32') { reject(new Error('Run the release producer on Linux, WSL2 or macOS.')); return; }
|
||||
const child = spawn(command, args, { cwd, env, detached: true, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
const stdout = [], stderr = []; let size = 0, overflow = false, settled = false, reapTimer;
|
||||
const terminate = () => {
|
||||
if (overflow) return;
|
||||
overflow = true;
|
||||
try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); }
|
||||
// An escaped descendant must never retain our pipes indefinitely.
|
||||
reapTimer = setTimeout(() => { child.stdout.destroy(); child.stderr.destroy(); finish(-1); }, 500);
|
||||
};
|
||||
const timer = setTimeout(terminate, timeout);
|
||||
const collect = (chunks) => (data) => { size += data.length; if (size > 64 * 2 ** 20) terminate(); else chunks.push(data); };
|
||||
child.stdout.on('data', collect(stdout)); child.stderr.on('data', collect(stderr));
|
||||
child.stdin.on('error', () => {}); child.stdin.end(input);
|
||||
child.on('error', () => { settled = true; clearTimeout(timer); clearTimeout(reapTimer); reject(new Error(`Required maintainer command ${basename(command)} is unavailable.`)); });
|
||||
function finish(code) {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer); clearTimeout(reapTimer);
|
||||
if (!quiet) writeFileSync(log, Buffer.concat([...stdout, ...stderr]), { mode: 0o600 });
|
||||
if (code !== 0 || overflow) reject(new Error(`${basename(command)} failed${quiet ? '.' : `; inspect private log ${log}`}`));
|
||||
else accept(Buffer.concat(stdout).toString());
|
||||
}
|
||||
child.on('close', finish);
|
||||
});
|
||||
};
|
||||
}
|
||||
function acquireOutput(output) {
|
||||
if (!existsSync(output)) mkdirSync(output, { mode: 0o700 });
|
||||
if (realpathSync(output) !== output || !statSync(output).isDirectory() || (statSync(output).mode & 0o077)) throw new Error("Use a canonical owner-only output directory.");
|
||||
if (!existsSync(join(output, 'publication-state.json')) && readdirSync(output).length) throw new Error("Choose an empty output directory or the matching previous publication directory.");
|
||||
const lock = join(output, '.publisher.lock');
|
||||
if (existsSync(lock)) {
|
||||
const pid = Number(readFileSync(lock, 'utf8'));
|
||||
if (!Number.isInteger(pid) || pid <= 0) throw new Error("Inspect the incomplete publisher lock before retrying.");
|
||||
let alive = true;
|
||||
try { process.kill(pid, 0); } catch (error) { if (error.code === 'ESRCH') alive = false; }
|
||||
if (alive) throw new Error("Another publisher owns this output directory.");
|
||||
rmSync(lock);
|
||||
}
|
||||
const fd = openSync(lock, 'wx', 0o600); writeFileSync(fd, String(process.pid)); closeSync(fd);
|
||||
return () => rmSync(lock, { force: true });
|
||||
}
|
||||
export async function publishInstallation(options) {
|
||||
const unlock = acquireOutput(options.output);
|
||||
const logs = join(options.output, 'logs'); mkdirSync(logs, { recursive: true, mode: 0o700 });
|
||||
const run = commandRunner(logs);
|
||||
let worktree, temporary;
|
||||
try {
|
||||
const revision = (await run('git', ['rev-parse', '--verify', `${options.revision}^{commit}`], { quiet: true })).trim();
|
||||
if (!/^[a-f0-9]{40}$/.test(revision)) throw new Error("Source revision is not a commit.");
|
||||
const identity = { revision, version: options.version, namespace: options.namespace, platforms: options.platforms, repository: options.repository };
|
||||
const statePath = join(options.output, 'publication-state.json');
|
||||
let state = { identity };
|
||||
if (existsSync(statePath)) {
|
||||
state = JSON.parse(readFileSync(statePath, 'utf8'));
|
||||
if (JSON.stringify(state.identity) !== JSON.stringify(identity)) throw new Error("Output directory belongs to a different release; choose a new directory.");
|
||||
}
|
||||
const save = () => { const temp = statePath + '.tmp'; writeFileSync(temp, JSON.stringify(state, null, 2) + '\n', { mode: 0o600 }); renameSync(temp, statePath); };
|
||||
save();
|
||||
console.log(`Release ${identity.version}: ${identity.namespace}, ${identity.platforms.join(',')}, source ${revision}`);
|
||||
await run('docker', ['info', '--format', '{{.OSType}}']);
|
||||
await run('docker', ['buildx', 'version']);
|
||||
const registry = dockerHub(await dockerCredentials(run));
|
||||
const hosting = await giteaHosting({ run, repository: options.repository, identity, body: `Installer prerelease for ${identity.platforms.join(', ')}.\n\nImages: docker.io/${identity.namespace}/thothii-core:${identity.version} and docker.io/${identity.namespace}/thothii-frontend:${identity.version}.\n\nDownload the native operator bundle and SHA256SUMS.txt below. This release supplies images, document validation and preflight; complete non-interactive setup and Windows/Omarchy/macOS acceptance are subsequent tickets. No example databases, credentials or user workspace data are included.` });
|
||||
async function prepare() {
|
||||
if (state.assets) {
|
||||
const names = [...identity.platforms.map((platform) => `thothii-${identity.version}-${platform.replace('/', '-')}.tar.gz`), 'SHA256SUMS.txt'];
|
||||
if (state.assets.length !== names.length) throw new Error("Cached artifact set is incomplete.");
|
||||
for (const asset of state.assets) if (!names.includes(asset.name) || asset.path !== join(options.output, asset.name) || !existsSync(asset.path) || sha256(readFileSync(asset.path)) !== asset.sha256) throw new Error("Previously built release artifact changed; do not overwrite an immutable version.");
|
||||
for (const [platform, images] of Object.entries(state.images)) for (const reference of Object.values(images)) {
|
||||
const [repository, digest] = reference.replace(/^docker.io\//, '').split('@');
|
||||
await registry.inspect(repository, digest, platform, { anonymous: true });
|
||||
}
|
||||
return state.assets;
|
||||
}
|
||||
temporary = realpathSync(mkdtempSync(join(tmpdir(), 'thothii-release-')));
|
||||
worktree = join(temporary, 'source');
|
||||
await run('git', ['worktree', 'add', '--detach', worktree, revision]);
|
||||
const sourceCompose = parse(readFileSync(join(worktree, 'compose.yaml'), 'utf8'));
|
||||
for (const role of ['core', 'frontend']) {
|
||||
console.log(`Preparing public repository ${identity.namespace}/thothii-${role}`);
|
||||
await registry.ensurePublic(identity.namespace, `thothii-${role}`);
|
||||
let existing = null;
|
||||
for (const platform of identity.platforms) {
|
||||
const image = await registry.inspect(`${identity.namespace}/thothii-${role}`, identity.version, platform, { allowMissing: true });
|
||||
if (image && (image.labels['org.opencontainers.image.revision'] !== revision || image.labels['org.opencontainers.image.version'] !== identity.version)) throw new Error("Image tag already belongs to another immutable build; select a new release version.");
|
||||
existing = existing || image;
|
||||
}
|
||||
if (!existing) {
|
||||
console.log(`Building and publishing ${role} (${identity.platforms.join(', ')})`);
|
||||
await run('docker', ['buildx', 'build', '--platform', identity.platforms.join(','), '--file', `docker/${role}.Dockerfile`, '--tag', `docker.io/${identity.namespace}/thothii-${role}:${identity.version}`, '--build-arg', `IMAGE_VERSION=${identity.version}`, '--label', `org.opencontainers.image.revision=${revision}`, '--label', `org.opencontainers.image.source=${identity.repository}`, '--provenance=false', '--sbom=false', '--push', '.'], { cwd: worktree, timeout: 45 * 60_000 });
|
||||
}
|
||||
}
|
||||
state.images = {};
|
||||
for (const platform of identity.platforms) {
|
||||
const images = {};
|
||||
for (const role of ['core', 'frontend']) images[role] = (await registry.inspect(`${identity.namespace}/thothii-${role}`, identity.version, platform, { anonymous: true })).reference;
|
||||
for (const [role, service] of [['catalog', 'catalog-db'], ['qdrant', 'qdrant'], ['embedding', 'embedding']]) {
|
||||
const [named, digest] = sourceCompose.services[service].image.split('@');
|
||||
let repository = named.replace(/:[^/:]+$/, '').replace(/^docker.io\//, '');
|
||||
if (!repository.includes('/')) repository = 'library/' + repository;
|
||||
images[role] = (await registry.inspect(repository, digest, platform, { anonymous: true })).reference;
|
||||
}
|
||||
state.images[platform] = images;
|
||||
}
|
||||
save();
|
||||
console.log('Building native operator bundles from the selected source');
|
||||
await run('npm', ['ci'], { cwd: join(worktree, 'backend'), timeout: 10 * 60_000 });
|
||||
const sourceTime = (await run('git', ['show', '-s', '--format=%cI', revision], { quiet: true })).trim();
|
||||
const targets = identity.platforms.map((platform) => platform.replace('/', '-'));
|
||||
await run('node', [join(repositoryRoot, 'backend/scripts/build-workspace-tools.mjs'), ...targets], { cwd: join(worktree, 'backend'), env: { ...process.env, THT_BUILD_SOURCE_ROOT: worktree, THT_BUILD_VERSION: identity.version, THT_BUILD_TIME: sourceTime }, timeout: 10 * 60_000 });
|
||||
const assets = [];
|
||||
for (const platform of identity.platforms) {
|
||||
const target = platform.replace('/', '-');
|
||||
const name = `thothii-${identity.version}-${target}`;
|
||||
const bundle = join(options.output, name);
|
||||
if (existsSync(bundle)) rmSync(bundle, { recursive: true }); // owned staging, never an installed runtime
|
||||
mkdirSync(bundle);
|
||||
prepareBundle({ source: worktree, destination: bundle, platform, version: identity.version, revision, images: state.images[platform] });
|
||||
mkdirSync(join(bundle, 'bin'));
|
||||
for (const executable of ['tht', 'tht-workspace-documents']) {
|
||||
copyFileSync(join(worktree, 'dist/workspace-tools', target, executable), join(bundle, 'bin', executable));
|
||||
chmodSync(join(bundle, 'bin', executable), 0o755);
|
||||
}
|
||||
// Resolve source-independent resource/config shape without any operator credentials.
|
||||
await run('docker', ['compose', '-f', join(bundle, 'compose.yaml'), '-f', join(bundle, 'deploy/compose.local.yaml'), 'config', '--no-interpolate', '--no-env-resolution', '--format', 'json']);
|
||||
const archive = join(options.output, name + '.tar.gz');
|
||||
await run('tar', ['-czf', archive, '-C', options.output, name]);
|
||||
assets.push({ name: basename(archive), path: archive, bytes: statSync(archive).size, sha256: sha256(readFileSync(archive)) });
|
||||
}
|
||||
const sums = join(options.output, 'SHA256SUMS.txt');
|
||||
writeFileSync(sums, assets.map((asset) => `${asset.sha256} ${asset.name}\n`).join(''));
|
||||
assets.push({ name: 'SHA256SUMS.txt', path: sums, bytes: statSync(sums).size, sha256: sha256(readFileSync(sums)) });
|
||||
// An empty Docker configuration proves the consumer can pull without publisher credentials.
|
||||
const publicConfig = join(temporary, 'public-docker'); mkdirSync(publicConfig);
|
||||
for (const [platform, images] of Object.entries(state.images)) {
|
||||
for (const reference of Object.values(images)) {
|
||||
console.log(`Verifying anonymous pull ${reference.split('@')[0]} (${platform})`);
|
||||
await run('docker', ['--config', publicConfig, 'pull', '--platform', platform, reference], { timeout: 20 * 60_000 });
|
||||
}
|
||||
console.log(`Smoke checking published images (${platform})`);
|
||||
await run('docker', ['run', '--rm', '--platform', platform, '--network', 'none', '--entrypoint', '/bin/sh', images.core, '-ec', 'test "$(pi --version)" = "$PI_VERSION"; tht --help >/dev/null; test -f /app/backend/dist/catalog/migrate.js; test -x /app/docker/workspace-maintenance-entrypoint.sh; test -f /app/docker/catalog-migrate.sh; test ! -e /run/secrets/thothii.secrets'], { timeout: 5 * 60_000 });
|
||||
await run('docker', ['run', '--rm', '--platform', platform, '--network', 'none', '--entrypoint', '/usr/local/bin/frontend-config-smoke', images.frontend], { timeout: 60_000 });
|
||||
}
|
||||
state.assets = assets; save();
|
||||
return assets;
|
||||
}
|
||||
const published = await publishVerifiedRelease({ hosting, prepare });
|
||||
state.releaseURL = published.html_url; state.complete = true; save();
|
||||
console.log(`Published and verified: ${published.html_url}`);
|
||||
return published;
|
||||
} finally {
|
||||
if (worktree && existsSync(worktree)) await run('git', ['worktree', 'remove', '--force', worktree]).catch(() => {});
|
||||
if (temporary && !existsSync(worktree ?? '')) rmSync(temporary, { recursive: true, force: true });
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
try { await publishInstallation(optionsFromArgs(process.argv.slice(2))); }
|
||||
catch (error) { console.error(error instanceof SyntaxError ? 'Invalid release metadata; no secret values are printed.' : error.message); process.exitCode = 1; }
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, readFileSync, writeFileSync, lstatSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { parse, stringify } from "yaml";
|
||||
|
||||
export const serviceRoles = Object.freeze({ core: "core", frontend: "frontend", "catalog-db": "catalog", "catalog-migrate": "core", "workspace-maintenance": "core", qdrant: "qdrant", embedding: "embedding", "embedding-model-init": "embedding" });
|
||||
export const sha256 = (data) => createHash("sha256").update(data).digest("hex");
|
||||
|
||||
/** Only distribution assets enter the bundle: never a checkout, environment file or workspace. */
|
||||
export function prepareBundle({ source, destination, platform, version, revision, images }) {
|
||||
const compose = parse(readFileSync(join(source, "compose.yaml"), "utf8"));
|
||||
if (Object.keys(compose.services).sort().join() !== Object.keys(serviceRoles).sort().join()) throw new Error("Release service contract changed; review packaging before publishing.");
|
||||
for (const [name, role] of Object.entries(serviceRoles)) {
|
||||
if (!/^docker\.io\/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$/.test(images[role] ?? "")) throw new Error("Release requires immutable Docker Hub image references.");
|
||||
delete compose.services[name].build;
|
||||
delete compose.services[name].pull_policy;
|
||||
compose.services[name].image = images[role];
|
||||
compose.services[name].platform = platform;
|
||||
}
|
||||
const files = {};
|
||||
const write = (name, data) => {
|
||||
mkdirSync(dirname(join(destination, name)), { recursive: true });
|
||||
writeFileSync(join(destination, name), data);
|
||||
files[name] = sha256(data);
|
||||
};
|
||||
write("compose.yaml", stringify(compose));
|
||||
for (const name of ["deploy/compose.local.yaml", "deploy/compose.git-https.yaml", "deploy/compose.git-ssh.yaml", "docker/catalog-db-init.sql", "docker/embedding-model-init.sh"]) {
|
||||
const path = join(source, name);
|
||||
if (!lstatSync(path).isFile()) throw new Error("Release asset must be a regular tracked file.");
|
||||
write(name, readFileSync(path));
|
||||
}
|
||||
// Any future source bind mount must be deliberately added to the asset allowlist.
|
||||
for (const service of Object.values(compose.services)) {
|
||||
for (const volume of service.volumes ?? []) {
|
||||
const sourcePath = typeof volume === "string" ? volume.split(":")[0] : volume.type === "bind" ? volume.source : undefined;
|
||||
if (sourcePath?.startsWith(".") && !files[sourcePath.replace(/^\.\//, "")]) throw new Error("Source bind mount is missing from the release bundle.");
|
||||
}
|
||||
}
|
||||
const manifest = { schema_version: 1, version, revision, validator_protocol: 1,
|
||||
requirements: { cpus: 2, memory_bytes: 4 * 2 ** 30, disk_bytes: 10 * 2 ** 30 },
|
||||
components: ["pi", "catalog-migrations", "workspace-maintenance"],
|
||||
images: Object.fromEntries(Object.entries(images).map(([role, reference]) => [role, { [platform]: reference }])),
|
||||
files, compose: ["compose.yaml", "deploy/compose.local.yaml"] };
|
||||
writeFileSync(join(destination, "release-manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
||||
writeFileSync(join(destination, "README.md"), `# ThothII ${version} — ${platform}\n\nSource / Sorgente: ${revision}\n\nThis prerelease provides images and document/preflight tools. Non-interactive execution and real-host acceptance are separate follow-up tickets; this is not a certified complete installation.\nQuesta prerelease fornisce immagini e strumenti di preparazione/preflight. Esecuzione non interattiva e collaudi reali sono incrementi successivi: non è ancora un'installazione completa certificata.\n\nUse bin/tht and its sibling bin/tht-workspace-documents together; no Node, Python, Bun or application checkout is required on the consumer host.\nWindows: use the Linux amd64 bundle inside Ubuntu WSL2, not a native Windows shell.\n\n1. bin/tht workspace prepare --directory NEW_WORKSPACE --id practice --name Practice\n2. bin/tht workspace validate --directory WORKSPACE\n3. bin/tht installation prepare --directory NEW_PRIVATE_INSTALLATION\n4. bin/tht installation preflight --directory INSTALLATION --release ABSOLUTE_RELEASE_DIR/release-manifest.json\n5. Complete the commented documents, generate technical credentials explicitly, then run installation validate and installation plan with --installation ABSOLUTE_INSTALLATION_FILE.\n\nImages are pinned by digest; runtime credentials and user workspaces are never bundled.\nConsult the accompanying IT/EN guides for prepared documents and mandatory runtime checks.\n`);
|
||||
for (const [name, target] of [["standalone-manual-it.md", "GUIDE-IT.md"], ["standalone-manual-en.md", "GUIDE-EN.md"], ["installation-preflight.md", "PREFLIGHT.md"]]) writeFileSync(join(destination, target), readFileSync(join(source, "docs/install", name)));
|
||||
return manifest;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve, join } from "node:path";
|
||||
import { parse } from "yaml";
|
||||
import { prepareBundle } from "./release-bundle.mjs";
|
||||
|
||||
test("consumer bundle pins every service and carries all source bind resources", () => {
|
||||
const output = mkdtempSync(join(tmpdir(), "thoth-release-test-"));
|
||||
const images = Object.fromEntries(["core", "frontend", "catalog", "qdrant", "embedding"].map((role) => [role, `docker.io/tylconsulting/${role}@sha256:${"a".repeat(64)}`]));
|
||||
try {
|
||||
prepareBundle({ source: resolve(import.meta.dirname, "../.."), destination: output, platform: "linux/amd64", version: "0.1.0-install-preview.1", revision: "b".repeat(40), images });
|
||||
const manifest = JSON.parse(readFileSync(join(output, "release-manifest.json")));
|
||||
const compose = parse(readFileSync(join(output, "compose.yaml"), "utf8"));
|
||||
assert.equal(compose.services.core.image, images.core);
|
||||
assert.equal(compose.services["catalog-migrate"].image, images.core);
|
||||
assert.equal(compose.services["workspace-maintenance"].image, images.core);
|
||||
assert.equal(compose.services["embedding-model-init"].image, images.embedding);
|
||||
for (const service of Object.values(compose.services)) {
|
||||
assert.equal(service.build, undefined);
|
||||
assert.equal(service.pull_policy, undefined);
|
||||
assert.equal(service.platform, "linux/amd64");
|
||||
for (const volume of service.volumes ?? []) {
|
||||
if (typeof volume === "string" && volume.startsWith("./")) assert.ok(existsSync(join(output, volume.split(":")[0])));
|
||||
}
|
||||
}
|
||||
assert.equal(manifest.validator_protocol, 1);
|
||||
assert.deepEqual(manifest.compose, ["compose.yaml", "deploy/compose.local.yaml"]);
|
||||
assert.ok(manifest.files["docker/catalog-db-init.sql"]);
|
||||
assert.ok(manifest.files["docker/embedding-model-init.sh"]);
|
||||
assert.ok(!existsSync(join(output, "backend")));
|
||||
assert.ok(!existsSync(join(output, "harness")));
|
||||
} finally { rmSync(output, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { boundedFetch, readLimited } from "./release-registry.mjs";
|
||||
import { sha256 } from "./release-bundle.mjs";
|
||||
|
||||
export async function giteaHosting({ run, repository, identity, body }) {
|
||||
const remote = new URL(repository);
|
||||
const credential = await run("git", ["credential", "fill"], { input: `protocol=https\nhost=${remote.host}\n\n`, quiet: true, env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } });
|
||||
const fields = Object.fromEntries(credential.trim().split("\n").map((line) => { const at = line.indexOf("="); return [line.slice(0, at), line.slice(at + 1)]; }));
|
||||
if (!fields.username || !fields.password) throw new Error("Gitea publishing credentials are unavailable in the Git credential store.");
|
||||
const authorization = `Basic ${Buffer.from(`${fields.username}:${fields.password}`).toString("base64")}`;
|
||||
const api = `${remote.origin}/api/v1/repos${remote.pathname.replace(/\.git$/, "")}`;
|
||||
const marker = `<!-- thothii-release:${JSON.stringify(identity)} -->`;
|
||||
const tag = `installation-v${identity.version}`;
|
||||
async function request(path, options = {}, allowMissing = false) {
|
||||
const response = await boundedFetch(api + path, { ...options, headers: { Authorization: authorization, ...options.headers } }, 120_000);
|
||||
if (allowMissing && response.status === 404) return null;
|
||||
if (!response.ok) throw new Error(`Gitea release operation failed (HTTP ${response.status}).`);
|
||||
const bytes = await readLimited(response, 4 * 2 ** 20);
|
||||
try { return JSON.parse(bytes.toString()); } catch { throw new Error("Gitea returned invalid release metadata."); }
|
||||
}
|
||||
const json = (method, value) => ({ method, headers: { "Content-Type": "application/json" }, body: JSON.stringify(value) });
|
||||
async function verifyTag(required = false) {
|
||||
const existing = await request(`/tags/${encodeURIComponent(tag)}`, {}, true);
|
||||
if ((!existing && required) || (existing && existing.commit?.sha !== identity.revision)) throw new Error("Release Git tag does not match the requested source revision.");
|
||||
}
|
||||
async function assets(release) { return request(`/releases/${release.id}/assets`); }
|
||||
async function verifyAsset(asset, expected, publicRead) {
|
||||
const url = new URL(asset.browser_download_url);
|
||||
if (url.origin !== remote.origin) throw new Error("Unexpected release asset origin.");
|
||||
const response = await boundedFetch(url, { headers: publicRead ? {} : { Authorization: authorization } }, 120_000);
|
||||
if (!response.ok || sha256(await readLimited(response, expected.bytes + 1)) !== expected.sha256) throw new Error("Published release asset does not match its verified checksum.");
|
||||
}
|
||||
return {
|
||||
async open() {
|
||||
const info = await request("");
|
||||
if (info.private || !info.permissions?.push) throw new Error("Release hosting must be a public Gitea repository with publication rights.");
|
||||
await verifyTag();
|
||||
const existing = await request(`/releases/tags/${encodeURIComponent(tag)}`, {}, true);
|
||||
if (existing) {
|
||||
if (!existing.body?.includes(marker)) throw new Error("Release version already belongs to another source or publication identity; it will not be overwritten.");
|
||||
return existing;
|
||||
}
|
||||
return request("/releases", json("POST", { tag_name: tag, target_commitish: identity.revision, name: `ThothII ${identity.version}`, body: `${body}\n\n${marker}`, draft: true, prerelease: true }));
|
||||
},
|
||||
async upload(release, asset) {
|
||||
const matching = (await assets(release)).filter((item) => item.name === asset.name);
|
||||
if (matching.length > 1) throw new Error("Ambiguous release assets; no published files were replaced.");
|
||||
if (matching.length === 1) { await verifyAsset(matching[0], asset, false); return; }
|
||||
const data = readFileSync(asset.path);
|
||||
if (sha256(data) !== asset.sha256) throw new Error("Local release asset changed before upload.");
|
||||
const form = new FormData(); form.append("attachment", new Blob([data]), asset.name);
|
||||
await request(`/releases/${release.id}/assets?name=${encodeURIComponent(asset.name)}`, { method: "POST", body: form });
|
||||
},
|
||||
async verify(release, expected, publicRead) {
|
||||
await verifyTag(publicRead);
|
||||
const uploaded = await assets(release);
|
||||
if (uploaded.length !== expected.length) throw new Error("Release asset set is incomplete or contains unexpected files.");
|
||||
for (const asset of expected) {
|
||||
const matching = uploaded.filter((item) => item.name === asset.name);
|
||||
if (matching.length !== 1) throw new Error("Release asset missing or ambiguous.");
|
||||
await verifyAsset(matching[0], asset, publicRead);
|
||||
}
|
||||
},
|
||||
async publish(release) {
|
||||
await verifyTag();
|
||||
return request(`/releases/${release.id}`, json("PATCH", { draft: false }));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { giteaHosting } from './release-hosting.mjs';
|
||||
|
||||
test('release hosting refuses a pre-existing Git tag on another commit', async () => {
|
||||
const fetch = globalThis.fetch;
|
||||
const revision = 'a'.repeat(40);
|
||||
let writes = 0;
|
||||
globalThis.fetch = async (url, options) => {
|
||||
if (options.method) { writes++; return Response.json({ id: 1, draft: true }); }
|
||||
if (String(url).includes('/releases/tags/')) return new Response('', { status: 404 });
|
||||
if (String(url).includes('/tags/installation-v')) return Response.json({ commit: { sha: 'b'.repeat(40) } });
|
||||
return Response.json({ private: false, permissions: { push: true } });
|
||||
};
|
||||
try {
|
||||
const hosting = await giteaHosting({ run: async () => 'username=test\npassword=test\n', repository: 'https://example.test/owner/repo', identity: { revision, version: '1.0.0' }, body: '' });
|
||||
await assert.rejects(hosting.open(), /tag.*revision/i);
|
||||
assert.equal(writes, 0);
|
||||
} finally { globalThis.fetch = fetch; }
|
||||
});
|
||||
|
||||
test('matching Git tag is accepted and verified again before publishing', async () => {
|
||||
const fetch = globalThis.fetch;
|
||||
const identity = { revision: 'a'.repeat(40), version: '1.0.0' };
|
||||
let sha = identity.revision;
|
||||
let writes = 0;
|
||||
globalThis.fetch = async (url, options) => {
|
||||
if (options.method) { writes++; return Response.json({ id: 1, draft: false }); }
|
||||
if (String(url).includes('/releases/tags/')) return Response.json({ id: 1, draft: true, body: `<!-- thothii-release:${JSON.stringify(identity)} -->` });
|
||||
if (String(url).includes('/tags/')) return Response.json({ commit: { sha } });
|
||||
if (String(url).endsWith('/assets')) return Response.json([]);
|
||||
return Response.json({ private: false, permissions: { push: true } });
|
||||
};
|
||||
try {
|
||||
const hosting = await giteaHosting({ run: async () => 'username=test\npassword=test\n', repository: 'https://example.test/owner/repo', identity, body: '' });
|
||||
const release = await hosting.open();
|
||||
await hosting.verify(release, [], false);
|
||||
sha = 'b'.repeat(40);
|
||||
await assert.rejects(hosting.verify(release, [], false), /tag.*revision/i);
|
||||
await assert.rejects(hosting.publish(release), /tag.*revision/i);
|
||||
assert.equal(writes, 0);
|
||||
} finally { globalThis.fetch = fetch; }
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { commandRunner } from './publish-installation.mjs';
|
||||
|
||||
test('publisher timeout terminates descendants that retain output pipes', async () => {
|
||||
const logs = mkdtempSync(join(tmpdir(), 'release-process-test-'));
|
||||
try {
|
||||
const started = Date.now();
|
||||
await assert.rejects(commandRunner(logs)(process.execPath, ['-e', `
|
||||
require('child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 2500)'], {stdio: 'inherit'});
|
||||
setTimeout(() => {}, 2500);
|
||||
`], { timeout: 250, quiet: true }));
|
||||
assert.ok(Date.now() - started < 1800, 'timeout must not wait for the descendant to exit naturally');
|
||||
} finally { rmSync(logs, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
/** Drafts are the publication boundary. A partial build/upload is never a consumer release. */
|
||||
export async function publishVerifiedRelease({ hosting, prepare }) {
|
||||
const release = await hosting.open();
|
||||
const assets = await prepare();
|
||||
if (!release.draft) {
|
||||
await hosting.verify(release, assets, true);
|
||||
return release;
|
||||
}
|
||||
for (const asset of assets) await hosting.upload(release, asset);
|
||||
await hosting.verify(release, assets, false);
|
||||
const published = await hosting.publish(release);
|
||||
await hosting.verify(published, assets, true);
|
||||
return published;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { publishVerifiedRelease } from "./release-publication.mjs";
|
||||
|
||||
test("an interrupted preparation stays draft and retry publishes only after every artifact verifies", async () => {
|
||||
const events = [];
|
||||
let failing = true;
|
||||
const hosting = {
|
||||
open: async () => ({ id: 7, draft: true }),
|
||||
upload: async (_draft, asset) => events.push(`upload:${asset.name}`),
|
||||
verify: async (_draft, assets, publicRead) => events.push(`verify:${publicRead}:${assets.length}`),
|
||||
publish: async () => { events.push("publish"); return { html_url: "https://example.test/release" }; },
|
||||
};
|
||||
const prepare = async () => {
|
||||
if (failing) throw new Error("frontend build unavailable");
|
||||
return [{ name: "bundle.tar.gz" }, { name: "SHA256SUMS" }];
|
||||
};
|
||||
await assert.rejects(publishVerifiedRelease({ hosting, prepare }), /frontend/);
|
||||
assert.deepEqual(events, []);
|
||||
failing = false;
|
||||
await publishVerifiedRelease({ hosting, prepare });
|
||||
assert.deepEqual(events, ["upload:bundle.tar.gz", "upload:SHA256SUMS", "verify:false:2", "publish", "verify:true:2"]);
|
||||
});
|
||||
|
||||
test("a published version is verified without replacing any asset", async () => {
|
||||
const events = [];
|
||||
await publishVerifiedRelease({ prepare: async () => [{ name: "bundle.tar.gz" }], hosting: {
|
||||
open: async () => ({ id: 7, draft: false }),
|
||||
upload: async () => { throw new Error("overwrote a published version"); },
|
||||
publish: async () => { throw new Error("republished a version"); },
|
||||
verify: async (_release, _assets, publicRead) => events.push(publicRead),
|
||||
} });
|
||||
assert.deepEqual(events, [true]);
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { sha256 } from "./release-bundle.mjs";
|
||||
|
||||
const accept = "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json";
|
||||
export async function boundedFetch(url, options = {}, timeout = 30_000) {
|
||||
try { return await fetch(url, { ...options, redirect: options.redirect ?? "error", signal: AbortSignal.timeout(timeout) }); }
|
||||
catch { throw new Error("Release network request failed; retry with the same output directory."); }
|
||||
}
|
||||
export async function readLimited(response, maximum) {
|
||||
const chunks = []; let size = 0;
|
||||
for await (const chunk of response.body) { size += chunk.length; if (size > maximum) throw new Error("Release response exceeded its bound."); chunks.push(chunk); }
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
async function jsonResponse(response, description) {
|
||||
if (!response.ok) throw new Error(`${description} refused (HTTP ${response.status}).`);
|
||||
const bytes = await readLimited(response, 4 * 2 ** 20);
|
||||
try { return { bytes, value: JSON.parse(bytes.toString()) }; }
|
||||
catch { throw new Error("Release service returned invalid metadata."); }
|
||||
}
|
||||
export async function dockerCredentials(run) {
|
||||
const config = JSON.parse(readFileSync(join(process.env.DOCKER_CONFIG || join(homedir(), ".docker"), "config.json"), "utf8"));
|
||||
const server = "https://index.docker.io/v1/";
|
||||
const helper = config.credHelpers?.[server] || config.credsStore;
|
||||
if (!helper || !/^[A-Za-z0-9._-]+$/.test(helper)) throw new Error("Use docker login with an OS credential store before publishing.");
|
||||
const auth = JSON.parse(await run(`docker-credential-${helper}`, ["get"], { input: server + "\n", quiet: true }));
|
||||
if (!auth.Username || !auth.Secret) throw new Error("Docker Hub login is unavailable.");
|
||||
return auth;
|
||||
}
|
||||
export function dockerHub(auth) {
|
||||
async function token(repository, anonymous) {
|
||||
const url = new URL("https://auth.docker.io/token");
|
||||
url.searchParams.set("service", "registry.docker.io");
|
||||
url.searchParams.set("scope", `repository:${repository}:pull`);
|
||||
const response = await boundedFetch(url, { headers: anonymous ? {} : { Authorization: `Basic ${Buffer.from(`${auth.Username}:${auth.Secret}`).toString("base64")}` } });
|
||||
return (await jsonResponse(response, "Registry authentication")).value.token;
|
||||
}
|
||||
async function registryJSON(repository, route, anonymous, allowMissing = false) {
|
||||
const bearer = await token(repository, anonymous);
|
||||
let response = await boundedFetch(`https://registry-1.docker.io/v2/${repository}/${route}`, { redirect: "manual", headers: { Accept: accept, Authorization: `Bearer ${bearer}` } });
|
||||
if ([302, 307].includes(response.status) && route.startsWith("blobs/")) {
|
||||
const target = new URL(response.headers.get("location"));
|
||||
if (target.protocol !== "https:" || target.username || target.password) throw new Error("Invalid registry blob redirect.");
|
||||
// Signed blob URLs are fetched without forwarding registry credentials.
|
||||
response = await boundedFetch(target);
|
||||
}
|
||||
if (allowMissing && response.status === 404) return null;
|
||||
const { bytes, value } = await jsonResponse(response, "Registry read");
|
||||
const digest = `sha256:${sha256(bytes)}`;
|
||||
const advertised = response.headers.get("docker-content-digest");
|
||||
if (advertised && advertised !== digest) throw new Error("Registry content digest mismatch.");
|
||||
return { value, digest };
|
||||
}
|
||||
return {
|
||||
async ensurePublic(namespace, name) {
|
||||
const response = await boundedFetch("https://hub.docker.com/v2/auth/token", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ identifier: auth.Username, secret: auth.Secret }) });
|
||||
const bearer = (await jsonResponse(response, "Docker Hub authentication")).value.access_token;
|
||||
const headers = { Authorization: `Bearer ${bearer}`, "Content-Type": "application/json" };
|
||||
const path = `https://hub.docker.com/v2/namespaces/${namespace}/repositories`;
|
||||
let existing = await boundedFetch(`${path}/${name}`, { headers });
|
||||
if (existing.status === 404) {
|
||||
existing = await boundedFetch(path, { method: "POST", headers, body: JSON.stringify({ namespace, name, registry: "docker.io", is_private: false, description: `ThothII ${name.endsWith("core") ? "core with embedded Pi" : "standalone frontend"}` }) });
|
||||
}
|
||||
const data = (await jsonResponse(existing, "Public repository preparation")).value;
|
||||
if (data.is_private !== false) throw new Error("Selected Docker Hub repository is private; make this release repository public before retrying.");
|
||||
},
|
||||
async inspect(repository, reference, platform, { anonymous = false, allowMissing = false } = {}) {
|
||||
let image = await registryJSON(repository, `manifests/${reference}`, anonymous, allowMissing);
|
||||
if (!image) return null;
|
||||
if (reference.startsWith("sha256:") && image.digest !== reference) throw new Error("Requested image digest does not match registry content.");
|
||||
if (image.value.manifests) {
|
||||
const match = image.value.manifests.find((entry) => `${entry.platform?.os}/${entry.platform?.architecture}` === platform);
|
||||
if (!match || !/^sha256:[a-f0-9]{64}$/.test(match.digest)) throw new Error("Image does not contain the requested platform.");
|
||||
image = await registryJSON(repository, `manifests/${match.digest}`, anonymous);
|
||||
if (image.digest !== match.digest) throw new Error("Image index digest mismatch.");
|
||||
}
|
||||
if (reference.startsWith("sha256:") && !image.value.config) throw new Error("Image metadata is incomplete.");
|
||||
const configDigest = image.value.config?.digest;
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(configDigest ?? "")) throw new Error("Image configuration digest is invalid.");
|
||||
const config = await registryJSON(repository, `blobs/${configDigest}`, anonymous);
|
||||
if (config.digest !== configDigest || `${config.value.os}/${config.value.architecture}` !== platform) throw new Error("Image configuration or platform mismatch.");
|
||||
return { reference: `docker.io/${repository}@${image.digest}`, labels: config.value.config?.Labels ?? {} };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { dockerHub } from "./release-registry.mjs";
|
||||
import { sha256 } from "./release-bundle.mjs";
|
||||
|
||||
test("registry verifies pinned config/platform and does not forward auth to blob storage", async () => {
|
||||
const original = globalThis.fetch;
|
||||
const config = JSON.stringify({ os: "linux", architecture: "amd64", config: { Labels: { "org.opencontainers.image.revision": "b".repeat(40) } } });
|
||||
const configDigest = "sha256:" + sha256(config);
|
||||
const manifest = JSON.stringify({ schemaVersion: 2, config: { digest: configDigest } });
|
||||
const imageDigest = "sha256:" + sha256(manifest);
|
||||
const auth = [];
|
||||
globalThis.fetch = async (target, options) => {
|
||||
const url = new URL(target);
|
||||
if (url.hostname === "auth.docker.io") return new Response(JSON.stringify({ token: "registry-token" }));
|
||||
if (url.hostname === "blob.example.test") { auth.push(options.headers?.Authorization); return new Response(config); }
|
||||
if (url.pathname.includes("/blobs/")) return new Response(null, { status: 307, headers: { location: "https://blob.example.test/config" } });
|
||||
return new Response(manifest, { headers: { "docker-content-digest": imageDigest } });
|
||||
};
|
||||
try {
|
||||
const registry = dockerHub({ Username: "publisher", Secret: "PRIVATE_TOKEN" });
|
||||
const image = await registry.inspect("example/core", imageDigest, "linux/amd64", { anonymous: true });
|
||||
assert.equal(image.reference, `docker.io/example/core@${imageDigest}`);
|
||||
assert.deepEqual(auth, [undefined]);
|
||||
await assert.rejects(registry.inspect("example/core", imageDigest, "linux/arm64"), /platform mismatch/);
|
||||
await assert.rejects(registry.inspect("example/core", "sha256:" + "0".repeat(64), "linux/amd64"), /Requested image digest/);
|
||||
} finally { globalThis.fetch = original; }
|
||||
});
|
||||
+18
-1
@@ -7,6 +7,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
import { createMemoryCleanup } from "./catalog/memory-cleanup.js";
|
||||
import { PiProcessManager } from "./pi/pi-process-manager.js";
|
||||
import { SseHub } from "./sse/sse-hub.js";
|
||||
import { authenticateSession, captureAuthConfigSnapshot, configuredOrigin } from "./auth/auth.js";
|
||||
@@ -80,6 +81,8 @@ import { createProductionWorkspacePreprocessingService } from "./workspace-maint
|
||||
import type { WorkspacePreprocessingService } from "./workspaces/preprocessing-service.js";
|
||||
import { PreprocessingStateStore } from "./workspaces/preprocessing-state.js";
|
||||
import { workspacePreprocessingRoutes } from "./routes/workspace-preprocessing.js";
|
||||
import { memoryRoutes } from "./routes/memory.js";
|
||||
import { evidenceRoutes } from "./routes/evidence.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
@@ -93,7 +96,7 @@ export interface BuildAppDeps {
|
||||
workspaceDiagnoser?: WorkspaceDiagnoser;
|
||||
workspaceDatabaseTester?: WorkspaceDatabaseTester;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear">;
|
||||
workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear"> & Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
|
||||
catalogRepository?: CatalogRepository;
|
||||
catalogService?: CatalogService;
|
||||
catalogPostgresAccess?: CatalogPostgresAccess;
|
||||
@@ -265,6 +268,11 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
catalogSchemaIntrospector,
|
||||
catalogOperationCoordinator,
|
||||
config.catalogSyncTimeoutMs,
|
||||
createMemoryCleanup(tht as ThtRunner, {
|
||||
internalQdrantUrl: config.internalQdrantUrl, internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId, internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
}),
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
|
||||
@@ -493,7 +501,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
return maintenanceBarrier.status();
|
||||
});
|
||||
sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
|
||||
memoryRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry, runtime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
} });
|
||||
metaRoutes(app, { harnessDir: config.harnessDir, modelCatalog: runtimeModelCatalog });
|
||||
evidenceRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry,
|
||||
registryRoot: config.workspaceRegistry.root, hostRegistryRoot: config.evidenceHostRegistryRoot,
|
||||
service: workspacePreprocessingService });
|
||||
workspaceRoutes(app, {
|
||||
registry: workspaceRegistry,
|
||||
config: config.workspaceRegistry,
|
||||
|
||||
@@ -119,7 +119,9 @@ export function authenticateSession(deps: AuthDependencies): preHandlerHookHandl
|
||||
subject: session.subject,
|
||||
...(session.displayName === undefined ? {} : { displayName: session.displayName }),
|
||||
roles: session.roles,
|
||||
permissions: session.permissions,
|
||||
// Sessions can outlive a deployment that changes the role permission catalog.
|
||||
// resolve() has already checked validity, including current local user roles.
|
||||
permissions: rolesToPermissions(session.roles),
|
||||
isAdmin: session.roles.includes("admin"),
|
||||
};
|
||||
if (STATE_CHANGING_METHODS.has(request.method)) {
|
||||
|
||||
@@ -38,7 +38,7 @@ const MAX_MAPPED_GROUPS = 128;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
export const PERMISSION_CATALOG: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
const invalid = (): Error => new Error("authentication configuration is invalid");
|
||||
|
||||
@@ -33,7 +33,7 @@ const EMPTY_HKDF_SALT = Buffer.alloc(0);
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
const PERMISSIONS = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
] as const satisfies readonly Permission[];
|
||||
|
||||
const invalid = (): Error => new Error("auth_session_store_invalid");
|
||||
|
||||
@@ -5,7 +5,7 @@ export type Role = "user" | "admin";
|
||||
export type Permission =
|
||||
| "session.use" | "session.read_all" | "session.manage_all"
|
||||
| "settings.manage" | "workspace.manage" | "workspace.secrets.manage"
|
||||
| "database.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
| "database.manage" | "memory.manage" | "evidence.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
|
||||
export interface AuthenticationSessionConfig {
|
||||
regularTtlSeconds: number;
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { readFileSync, lstatSync } from "node:fs";
|
||||
import { parseAllDocuments } from "yaml";
|
||||
import { decode, DocumentError, runWorkspaceDocuments } from "../workspaces/documents.js";
|
||||
import { validateDatabaseBootstrap } from "./bootstrap-documents.js";
|
||||
|
||||
/** Internal sibling protocol: only references cross back to Go, never secret contents. */
|
||||
export function runBootstrapValidation(args: string[]): { status: number; output: string } {
|
||||
try {
|
||||
if (args.length !== 5 || args[0] !== "--directory" || args[2] !== "--bootstrap" || args[4] !== "--json") throw new Error("usage");
|
||||
const checked = runWorkspaceDocuments(["validate", "--directory", args[1], "--json"]);
|
||||
if (checked.status !== 0) return checked;
|
||||
const info = lstatSync(args[3]);
|
||||
if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024) throw new Error("file");
|
||||
const validated = decode(readFileSync(args[3], "utf8"), "database-bootstrap.yaml", (source) =>
|
||||
validateDatabaseBootstrap(parseAllDocuments(source)[0].toJSON(), args[1]), "database bootstrap schema v1 and the Catalog binding contract");
|
||||
return { status: 0, output: JSON.stringify({ schema_version: 1, ok: true, secret_files: validated.secretFiles, warnings: validated.warnings, issues: [] }) };
|
||||
} catch (error) {
|
||||
const issue = error instanceof DocumentError ? error.issue : { document: "database-bootstrap.yaml", field: "$", code: "bootstrap_invalid", correction: "Supply one complete Catalog database configuration per workspace in a readable local bootstrap document." };
|
||||
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, issues: [issue] }) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { z } from "zod";
|
||||
import { databaseConfigurationSchema } from "./configuration-schema.js";
|
||||
import { parseWorkspaceCatalogYaml } from "../workspaces/catalog.js";
|
||||
import { parseWorkspaceYaml } from "../workspaces/schema.js";
|
||||
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
|
||||
|
||||
const reference = z.string().min(1).max(4096);
|
||||
const database = databaseConfigurationSchema.extend({
|
||||
secretFiles: z.object({ password: reference.optional(), apiKey: reference.optional(), sshPrivateKey: reference.optional(), sshPrivateKeyPassphrase: reference.optional(), sshKnownHosts: reference.optional(), tlsCa: reference.optional() }).strict(),
|
||||
evidenceSecretFiles: z.object({ "evidence.signed_urls": reference.optional(), "evidence.access_key": reference.optional(), "evidence.secret_key": reference.optional(), "evidence.session_token": reference.optional() }).strict().optional(),
|
||||
});
|
||||
const bootstrap = z.object({ schemaVersion: z.literal(1), databases: z.array(database).min(1).max(1000) }).strict();
|
||||
|
||||
export const parseDatabaseBootstrap = (value: unknown) => bootstrap.parse(value);
|
||||
|
||||
export interface BootstrapReference { field: string; path: string }
|
||||
|
||||
/** Offline bootstrap boundary: runtime Catalog owns the resulting bindings after import. */
|
||||
export function validateDatabaseBootstrap(value: unknown, workspaceRoot: string): { secretFiles: BootstrapReference[]; warnings: string[] } {
|
||||
const document = bootstrap.parse(value);
|
||||
const catalog = parseWorkspaceCatalogYaml(readFileSync(join(workspaceRoot, "thoth-workspaces.yaml"), "utf8"));
|
||||
const expected = new Set(catalog.workspaces.map((entry) => entry.id));
|
||||
const seen = new Set<string>();
|
||||
const secretFiles: BootstrapReference[] = [];
|
||||
const warnings: string[] = [];
|
||||
const issue = (path: (string | number)[], message: string): never => { throw new z.ZodError([{ code: "custom", path, message }]); };
|
||||
document.databases.forEach((entry, index) => {
|
||||
if (!expected.has(entry.workspaceId) || seen.has(entry.workspaceId)) issue(["databases", index, "workspaceId"], "Declare each catalog workspace exactly once.");
|
||||
seen.add(entry.workspaceId);
|
||||
const required = entry.binding.transport === "rest_api"
|
||||
? entry.binding.restAuth === "none" ? [] : ["apiKey"] as const
|
||||
: entry.binding.transport === "ssh_tunnel" ? ["password", "sshPrivateKey", "sshKnownHosts"] as const : ["password"] as const;
|
||||
for (const name of required) {
|
||||
if (!entry.secretFiles[name]) issue(["databases", index, "secretFiles", name], "Supply a protected file reference for this transport.");
|
||||
}
|
||||
if (entry.binding.transport === "ssh_tunnel") warnings.push(`databases.${index}:ssh_tunnel supports Catalog diagnostics, not NL-to-SQL sessions; choose direct or REST for practice.`);
|
||||
for (const [name, path] of Object.entries(entry.secretFiles)) secretFiles.push({ field: `databases.${index}.secretFiles.${name}`, path });
|
||||
const workspace = parseWorkspaceYaml(readFileSync(join(workspaceRoot, entry.workspaceId, "workspace.yaml"), "utf8"));
|
||||
const requirements = discoverWorkspaceSecretRequirements(workspace, {});
|
||||
for (const requirement of requirements.filter((item) => item.connector === "evidence" && item.required)) {
|
||||
if (!(entry.evidenceSecretFiles as Record<string, string> | undefined)?.[requirement.id]) issue(["databases", index, "evidenceSecretFiles"], "Supply the configured Evidence authentication file references.");
|
||||
}
|
||||
for (const [name, path] of Object.entries(entry.evidenceSecretFiles ?? {})) secretFiles.push({ field: `databases.${index}.evidenceSecretFiles.${name}`, path });
|
||||
});
|
||||
if (seen.size !== expected.size) issue(["databases"], "Add a database binding for every catalog workspace.");
|
||||
return { secretFiles, warnings };
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3";
|
||||
import { parseWorkspaceYaml } from "../workspaces/schema.js";
|
||||
import { evidencePolicy } from "../workspaces/evidence/preprocessing.js";
|
||||
import type { parseDatabaseBootstrap } from "./bootstrap-documents.js";
|
||||
import { probePublicEvidenceUrl, publicEvidenceAgent } from "./evidence-probe-http.js";
|
||||
|
||||
type Entry = ReturnType<typeof parseDatabaseBootstrap>["databases"][number];
|
||||
|
||||
/** Read-only availability probes; domain correctness and materialization remain runtime gates. */
|
||||
export async function probeEvidence(entry: Entry, root: string): Promise<void> {
|
||||
const evidence = parseWorkspaceYaml(readFileSync(join(root, entry.workspaceId, "workspace.yaml"), "utf8")).evidence;
|
||||
if (!evidence || evidence.source.type === "filesystem") return;
|
||||
if (evidencePolicy(evidence)) throw new Error("Evidence egress policy refused");
|
||||
const secret = (name: keyof NonNullable<Entry["evidenceSecretFiles"]>) => {
|
||||
const path = entry.evidenceSecretFiles?.[name];
|
||||
if (!path) throw new Error("Evidence credential missing");
|
||||
return readFileSync(path, "utf8").trim();
|
||||
};
|
||||
const source = evidence.source;
|
||||
if (source.type === "http") {
|
||||
const urls: unknown = source.authentication === "signed_urls_file"
|
||||
? JSON.parse(secret("evidence.signed_urls")) : source.uris;
|
||||
if (!Array.isArray(urls) || urls.length !== source.uris.length || urls.length > 1000) throw new Error("Invalid signed URLs");
|
||||
for (const [index, value] of urls.entries()) {
|
||||
if (typeof value !== "string") throw new Error("Invalid signed URL");
|
||||
const url = new URL(value);
|
||||
const provenance = new URL(source.uris[index]);
|
||||
// Signed queries may authorize the same identity, never a different host/path.
|
||||
if (url.origin !== provenance.origin || url.pathname !== provenance.pathname || url.username || url.password || url.hash) throw new Error("Invalid signed URL identity");
|
||||
await probePublicEvidenceUrl(url);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// The shared runtime policy currently permits trusted AWS endpoints with explicit file credentials.
|
||||
const location = new URL(source.uri);
|
||||
const client = new S3Client({
|
||||
region: source.region ?? "us-east-1", maxAttempts: 1,
|
||||
requestHandler: { httpsAgent: publicEvidenceAgent(), connectionTimeout: 5_000, requestTimeout: 5_000 },
|
||||
credentials: { accessKeyId: secret("evidence.access_key"), secretAccessKey: secret("evidence.secret_key"),
|
||||
...(entry.evidenceSecretFiles?.["evidence.session_token"] ? { sessionToken: secret("evidence.session_token") } : {}) },
|
||||
});
|
||||
try {
|
||||
await client.send(new ListObjectsV2Command({ Bucket: location.hostname, Prefix: decodeURIComponent(location.pathname.slice(1)), MaxKeys: 1 }), { abortSignal: AbortSignal.timeout(5_000) });
|
||||
} finally { client.destroy(); }
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { parse } from "yaml";
|
||||
import { parseDatabaseBootstrap } from "./bootstrap-documents.js";
|
||||
import { runBootstrapValidation } from "./bootstrap-cli.js";
|
||||
import { createConcreteDiagnosticAdapters, type DiagnosticAdapters } from "../workspaces/diagnostics.js";
|
||||
import { probeEvidence } from "./bootstrap-evidence-probes.js";
|
||||
|
||||
interface ProbeCheck { id: string; outcome: "passed" | "error"; field: string; action: string }
|
||||
|
||||
/** Uses the same read-only, authenticated connector diagnostics as the Catalog. */
|
||||
export async function probeBootstrapDependencies(value: unknown, adapters: DiagnosticAdapters = createConcreteDiagnosticAdapters(), workspaceRoot?: string) {
|
||||
const document = parseDatabaseBootstrap(value);
|
||||
const checks: ProbeCheck[] = [];
|
||||
for (const [index, entry] of document.databases.entries()) {
|
||||
let outcome: ProbeCheck["outcome"] = "passed";
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 5_000);
|
||||
try {
|
||||
if (entry.binding.transport === "ssh_tunnel") throw new Error("session transport unavailable");
|
||||
await adapters.probeConnector({
|
||||
role: "dwh", transport: entry.binding.transport,
|
||||
host: entry.binding.host, port: entry.binding.port, user: entry.binding.username,
|
||||
baseUrl: entry.binding.baseUrl,
|
||||
credentialFile: entry.binding.transport === "rest_api" ? entry.secretFiles.apiKey : entry.secretFiles.password,
|
||||
tlsCaFile: entry.secretFiles.tlsCa, tlsServername: entry.binding.tlsServername,
|
||||
resource: { database: entry.databaseName, schema: entry.schema },
|
||||
timeoutMs: 5_000, signal: controller.signal,
|
||||
diagnostic: { method: "GET", path: entry.binding.restPath ?? "/health", auth: entry.binding.restAuth ?? "bearer" },
|
||||
});
|
||||
} catch { outcome = "error"; } finally { clearTimeout(timer); }
|
||||
checks.push({ id: `database-${index}`, outcome, field: `database-bootstrap.databases.${index}`,
|
||||
action: entry.binding.transport === "ssh_tunnel"
|
||||
? "Choose postgres_direct or rest_api for NL-to-SQL practice; SSH diagnostics alone cannot establish session readiness."
|
||||
: "Require an authenticated read-only connection and access to the configured database/schema; correct endpoint, permissions or protected credentials." });
|
||||
if (workspaceRoot) {
|
||||
let evidenceOutcome: ProbeCheck["outcome"] = "passed";
|
||||
try { await probeEvidence(entry, workspaceRoot); } catch { evidenceOutcome = "error"; }
|
||||
checks.push({ id: `evidence-${index}`, outcome: evidenceOutcome, field: `workspaces.${index}.evidence`, action: "Require readable local Evidence or authenticated bounded HTTP/S3 access under the canonical egress policy; domain meaning is verified during practice." });
|
||||
}
|
||||
}
|
||||
return { schema_version: 1, ok: checks.every((check) => check.outcome === "passed"), checks };
|
||||
}
|
||||
|
||||
export async function runBootstrapProbes(args: string[]) {
|
||||
const validation = runBootstrapValidation(args);
|
||||
if (validation.status !== 0) return validation;
|
||||
try {
|
||||
const report = await probeBootstrapDependencies(parse(readFileSync(args[3], "utf8")), undefined, args[1]);
|
||||
return { status: report.ok ? 0 : 1, output: JSON.stringify(report) };
|
||||
} catch {
|
||||
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, checks: [{ id: "database-probes", outcome: "error", field: "database-bootstrap", action: "Revalidate prepared documents and protected credential references." }] }) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { z } from "zod";
|
||||
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
|
||||
import { DATABASE_TRANSPORTS } from "./types.js";
|
||||
|
||||
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
|
||||
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
|
||||
const nonEmpty = z.string().trim().min(1).max(512);
|
||||
const port = z.number().int().min(1).max(65_535);
|
||||
const optionalText = nonEmpty.optional();
|
||||
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
|
||||
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
|
||||
const bindingSchema = z.object({
|
||||
transport: z.enum(DATABASE_TRANSPORTS),
|
||||
host: optionalText,
|
||||
port: port.optional(),
|
||||
username: optionalText,
|
||||
baseUrl: z.string().max(2048)
|
||||
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
|
||||
.optional(),
|
||||
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
|
||||
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
|
||||
tlsServername: optionalText,
|
||||
sshHost,
|
||||
sshPort: port.optional(),
|
||||
sshUsername,
|
||||
sshTargetHost: sshHost,
|
||||
sshTargetPort: port.optional(),
|
||||
}).strict().superRefine((binding, context) => {
|
||||
const required = binding.transport === "postgres_direct"
|
||||
? ["host", "port", "username"] as const
|
||||
: binding.transport === "rest_api"
|
||||
? ["baseUrl", "restPath", "restAuth"] as const
|
||||
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
|
||||
for (const field of required) {
|
||||
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
|
||||
}
|
||||
});
|
||||
export const databaseConfigurationSchema = z.object({
|
||||
workspaceId: workspaceIdSchema,
|
||||
engine: z.literal("postgres"),
|
||||
databaseName: identifier,
|
||||
schema: identifier,
|
||||
binding: bindingSchema,
|
||||
}).strict();
|
||||
@@ -0,0 +1,57 @@
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import { request as httpsRequest, Agent } from "node:https";
|
||||
import type { LookupFunction } from "node:net";
|
||||
import ipaddr from "ipaddr.js";
|
||||
|
||||
const refused = () => new Error("Evidence network policy refused");
|
||||
function normalizedPublicAddress(value: string): string {
|
||||
const address = ipaddr.process(value);
|
||||
if (address.range() !== "unicast") throw refused();
|
||||
return address.toString();
|
||||
}
|
||||
|
||||
/** Reject the entire DNS answer set, then pin the connection to that verified set. */
|
||||
export async function resolvePublicEvidenceHost(hostname: string) {
|
||||
const values = await lookup(hostname.replace(/^\[|\]$/g, ""), { all: true });
|
||||
if (!values.length) throw refused();
|
||||
values.forEach((value) => normalizedPublicAddress(value.address));
|
||||
return values;
|
||||
}
|
||||
const publicLookup: LookupFunction = (hostname, options, callback) => {
|
||||
void resolvePublicEvidenceHost(hostname).then((values) => {
|
||||
if (options.all) callback(null, values);
|
||||
else callback(null, values[0].address, values[0].family);
|
||||
}, () => callback(refused(), "", 0));
|
||||
};
|
||||
|
||||
// Node's direct agent does not inherit HTTP proxy environment or ambient credentials.
|
||||
export const publicEvidenceAgent = () => new Agent({ lookup: publicLookup });
|
||||
|
||||
export async function probePublicEvidenceUrl(url: URL): Promise<void> {
|
||||
if (!['http:', 'https:'].includes(url.protocol)) throw refused();
|
||||
const signal = AbortSignal.timeout(5_000);
|
||||
const values = await Promise.race([
|
||||
resolvePublicEvidenceHost(url.hostname),
|
||||
new Promise<never>((_, reject) => signal.addEventListener("abort", () => reject(refused()), { once: true })),
|
||||
]);
|
||||
signal.throwIfAborted();
|
||||
const allowed = new Set(values.map((value) => normalizedPublicAddress(value.address)));
|
||||
const pinned: LookupFunction = (_hostname, options, callback) => {
|
||||
if (options.all) callback(null, values);
|
||||
else callback(null, values[0].address, values[0].family);
|
||||
};
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(url, {
|
||||
method: "GET", lookup: pinned, signal, agent: false,
|
||||
}, (response) => {
|
||||
try {
|
||||
const peer = response.socket.remoteAddress;
|
||||
if (!peer || !allowed.has(normalizedPublicAddress(peer)) || !response.statusCode || response.statusCode < 200 || response.statusCode >= 300) throw refused();
|
||||
resolve();
|
||||
} catch { reject(refused()); } finally { response.destroy(); }
|
||||
});
|
||||
request.on("error", () => reject(refused()));
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
|
||||
import type { CatalogSyncRun, WorkspaceDatabase } from "./types.js";
|
||||
|
||||
/** Internal continuation of an applied physical sync, using the harness Memory boundary. */
|
||||
export function createMemoryCleanup(runner: Pick<ThtRunner, "withPrincipal">, runtime: SemanticRuntimeConfig) {
|
||||
return async (database: WorkspaceDatabase, run: CatalogSyncRun): Promise<number> => {
|
||||
if (run.phase !== "memory_cleanup" || !run.plannedDiff) throw new Error("Physical cleanup is not committed");
|
||||
const result = await runner.withPrincipal({ issuer: "installation", subject: "catalog-sync",
|
||||
roles: ["admin"], permissions: ["memory.manage"], isAdmin: true,
|
||||
}).runWithRuntimeSnapshot(["memory", "admin", "--workspace", database.workspaceId], JSON.stringify({
|
||||
action: "cleanup", runtime, request: { sync_id: run.id, database: database.databaseName,
|
||||
schema_name: database.schema, removed_tables: run.plannedDiff.deletedTables,
|
||||
removed_columns: run.plannedDiff.deletedColumns.map(column => ({ table: column.tableName, column: column.columnName })),
|
||||
},
|
||||
}));
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.code !== 0 || payload.indexed !== true || !Number.isInteger(payload.deleted) || payload.deleted < 0) {
|
||||
throw new Error("Memory cleanup is incomplete");
|
||||
}
|
||||
return payload.deleted;
|
||||
};
|
||||
}
|
||||
@@ -926,6 +926,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined> {
|
||||
const database = this.records.get(databaseId);
|
||||
if (!database || database.version !== expectedDatabaseVersion) return undefined;
|
||||
@@ -1141,6 +1142,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
if (scope === "all") {
|
||||
this.records.set(databaseId, { ...database, schemaSyncedVersion: expectedDatabaseVersion, schemaSyncedAt: now });
|
||||
}
|
||||
if (syncRunId) await this.updateSyncRun(syncRunId, { phase: "memory_cleanup" });
|
||||
return {
|
||||
tables: (await this.listTables(databaseId)).length,
|
||||
columns: [...this.columns.values()].filter((column) => this.tables.get(column.tableId)?.databaseId === databaseId).length,
|
||||
@@ -1254,7 +1256,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
for (const run of this.syncRuns.values()) {
|
||||
if (["queued", "running", "awaiting_confirmation", "applying"].includes(run.state)) {
|
||||
await this.updateSyncRun(run.id, {
|
||||
state: "interrupted", phase: "completed", finishedAt: new Date().toISOString(),
|
||||
state: "interrupted", phase: run.phase === "memory_cleanup" ? "memory_cleanup" : "completed", finishedAt: new Date().toISOString(),
|
||||
errorCode: "SYNC_INTERRUPTED", errorMessage: "Synchronization was interrupted by a service restart",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -102,7 +102,7 @@ export function loadMetadataGenerationModels(options: {
|
||||
...(apiKeyEnv ? { apiKeyEnv, apiKey } : {}),
|
||||
}));
|
||||
}
|
||||
const result = new RestartLoadedMetadataGenerationModels(models, catalog.defaultMetadataGeneration);
|
||||
const result = new RestartLoadedMetadataGenerationModels(models, models.size ? catalog.defaultInteraction : null);
|
||||
const safe = result.catalog();
|
||||
return {
|
||||
catalog: () => ({
|
||||
|
||||
@@ -1604,6 +1604,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined> {
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases").select("version")
|
||||
@@ -1779,6 +1780,10 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
schemaSyncedAt: now,
|
||||
}).where("id", "=", databaseId).execute();
|
||||
}
|
||||
if (syncRunId) {
|
||||
await trx.updateTable("catalogSyncRuns").set({ phase: "memory_cleanup" })
|
||||
.where("id", "=", syncRunId).where("databaseId", "=", databaseId).execute();
|
||||
}
|
||||
return {
|
||||
tables: snapshot.tables.length,
|
||||
columns: scope === "tables" ? undefined : snapshot.columns.length,
|
||||
@@ -1882,7 +1887,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
|
||||
async interruptActiveSyncRuns(): Promise<void> {
|
||||
await this.db.updateTable("catalogSyncRuns").set({
|
||||
state: "interrupted", phase: "completed", errorCode: "worker_restarted",
|
||||
state: "interrupted", phase: sql`case when phase='memory_cleanup' then phase else 'completed' end`, errorCode: "worker_restarted",
|
||||
errorMessage: "Synchronization was interrupted by a backend restart.",
|
||||
finishedAt: sql`now()`, updatedAt: sql`now()`,
|
||||
leaseOwner: null, leaseExpiresAt: null,
|
||||
|
||||
@@ -58,6 +58,8 @@ export class CatalogSyncWorker {
|
||||
private readonly introspector: CatalogSchemaIntrospector,
|
||||
private readonly operations: CatalogOperationCoordinator,
|
||||
private readonly timeoutMs: number,
|
||||
private readonly cleanupMemory: (database: WorkspaceDatabase, run: CatalogSyncRun) => Promise<number>
|
||||
= async () => 0,
|
||||
) {}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
@@ -67,6 +69,9 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
|
||||
async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise<CatalogSyncRun> {
|
||||
if ((await this.repository.listSyncRuns(database.id, 100)).some(run => run.phase === "memory_cleanup")) {
|
||||
throw new CatalogConflictError("Retry the pending Memory cleanup before starting another synchronization");
|
||||
}
|
||||
const uniqueTableIds = [...new Set(tableIds)];
|
||||
if (scope === "columns") {
|
||||
const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId)));
|
||||
@@ -109,7 +114,7 @@ export class CatalogSyncWorker {
|
||||
async cancel(runId: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = await this.repository.getSyncRun(runId);
|
||||
if (!run) return undefined;
|
||||
if (run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
|
||||
if (run.phase === "memory_cleanup" || run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
|
||||
await this.repository.requestSyncRunCancellation(runId);
|
||||
this.controllers.get(runId)?.abort();
|
||||
if (run.state === "queued" || run.state === "awaiting_confirmation") {
|
||||
@@ -138,6 +143,16 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
const database = await this.repository.get(previous.databaseId);
|
||||
if (!database) return undefined;
|
||||
if (previous.phase === "memory_cleanup") {
|
||||
const release = this.operations.reserve(database.id);
|
||||
try {
|
||||
const queued = await this.repository.updateSyncRun(runId, { state: "queued", cancelRequested: false,
|
||||
errorCode: null, errorMessage: null, finishedAt: null, leaseOwner: null, leaseExpiresAt: null });
|
||||
this.reservations.set(runId, release);
|
||||
this.launch(runId);
|
||||
return queued;
|
||||
} catch (error) { release(); throw error; }
|
||||
}
|
||||
return await this.start(database, previous.scope, previous.tableIds);
|
||||
}
|
||||
|
||||
@@ -179,6 +194,10 @@ export class CatalogSyncWorker {
|
||||
if (!database || database.version !== claimed.requestedDatabaseVersion) {
|
||||
throw new CatalogConflictError("Database binding changed before synchronization started");
|
||||
}
|
||||
if (claimed.phase === "memory_cleanup") {
|
||||
await this.finishMemoryCleanup(database, claimed);
|
||||
return;
|
||||
}
|
||||
const progress: CatalogSchemaScanProgress = async (phase, counts) => {
|
||||
await this.checkCancelled(runId);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
@@ -230,36 +249,30 @@ export class CatalogSyncWorker {
|
||||
claimed.scope,
|
||||
claimed.tableIds,
|
||||
snapshot,
|
||||
runId,
|
||||
);
|
||||
if (!applied) throw new CatalogConflictError("Database binding changed before schema changes were applied");
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: "succeeded",
|
||||
phase: "completed",
|
||||
counts: applied,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
confirmationToken: null,
|
||||
heartbeatAt: new Date().toISOString(),
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(runId, "info", "succeeded", "Synchronization completed.", { ...applied });
|
||||
this.release(runId);
|
||||
const cleanupRun = await this.repository.updateSyncRun(runId, { counts: applied });
|
||||
if (!cleanupRun) throw new Error("Synchronization run disappeared");
|
||||
await this.finishMemoryCleanup(database, cleanupRun);
|
||||
/* Completion is recorded only after the durable Memory cleanup succeeds. */
|
||||
return;
|
||||
} catch (error) {
|
||||
const current = await this.repository.getSyncRun(runId);
|
||||
const cancelled = !timedOut && (error instanceof SyncCancelledError || controller.signal.aborted || current?.cancelRequested);
|
||||
const failure = timedOut
|
||||
const failure = current?.phase === "memory_cleanup"
|
||||
? { code: "memory_cleanup_pending", message: "Catalog synchronized. Memory cleanup is pending; retry this synchronization to complete it." }
|
||||
: timedOut
|
||||
? { code: "schema_sync_timed_out", message: "Schema synchronization timed out." }
|
||||
: safeFailure(error);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: cancelled ? "cancelled" : "failed",
|
||||
phase: "completed",
|
||||
phase: current?.phase === "memory_cleanup" ? "memory_cleanup" : "completed",
|
||||
errorCode: cancelled ? null : failure.code,
|
||||
errorMessage: cancelled ? null : failure.message,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
observedSnapshot: current?.phase === "memory_cleanup" ? current.observedSnapshot : null,
|
||||
plannedDiff: current?.phase === "memory_cleanup" ? current.plannedDiff : null,
|
||||
confirmationToken: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
@@ -278,6 +291,20 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
}
|
||||
|
||||
private async finishMemoryCleanup(database: WorkspaceDatabase, run: CatalogSyncRun): Promise<void> {
|
||||
if (!run.plannedDiff || run.phase !== "memory_cleanup") throw new Error("Missing committed cleanup context");
|
||||
await this.repository.appendSyncEvent(run.id, "info", "memory_cleanup", "Removing Memory cards with deleted physical dependencies.");
|
||||
const memoryDeleted = await this.cleanupMemory(database, run);
|
||||
const counts = { ...run.counts, memoryDeleted };
|
||||
await this.repository.updateSyncRun(run.id, {
|
||||
state: "succeeded", phase: "completed", counts, finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null, plannedDiff: null, confirmationToken: null,
|
||||
heartbeatAt: new Date().toISOString(), leaseOwner: null, leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(run.id, "info", "succeeded", "Synchronization completed.", counts);
|
||||
this.release(run.id);
|
||||
}
|
||||
|
||||
private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void {
|
||||
const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const;
|
||||
for (const name of required) {
|
||||
|
||||
@@ -366,7 +366,7 @@ export type CatalogSyncState =
|
||||
export type CatalogSyncPhase =
|
||||
| "queued" | "connecting" | "scanning_tables" | "scanning_columns"
|
||||
| "scanning_relationships" | "planning" | "awaiting_confirmation"
|
||||
| "applying" | "completed";
|
||||
| "applying" | "memory_cleanup" | "completed";
|
||||
|
||||
export interface CatalogSchemaDiff {
|
||||
deletedTables: string[];
|
||||
@@ -375,6 +375,7 @@ export interface CatalogSchemaDiff {
|
||||
}
|
||||
|
||||
export interface CatalogSyncCounts {
|
||||
memoryDeleted?: number;
|
||||
tables?: number;
|
||||
columns?: number;
|
||||
relationships?: number;
|
||||
@@ -587,6 +588,7 @@ export interface CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined>;
|
||||
createSyncRun(
|
||||
databaseId: string,
|
||||
|
||||
@@ -30,8 +30,11 @@ export interface AppConfig {
|
||||
dataRoot?: string;
|
||||
ollamaEnsureTimeoutMs: number;
|
||||
piManagementTimeoutMs: number;
|
||||
/** Host CLI platform projected into Docker; not the browser or container OS. */
|
||||
hostPlatform?: string;
|
||||
secretsFile?: string;
|
||||
installationConfigFile?: string;
|
||||
evidenceHostRegistryRoot?: string;
|
||||
modelCatalogFile?: string;
|
||||
sensitivityNer?: {
|
||||
pythonExecutable: string;
|
||||
@@ -474,8 +477,10 @@ export function loadConfig(
|
||||
dataRoot: env.THT_DATA_ROOT,
|
||||
ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000),
|
||||
piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
|
||||
hostPlatform: env.THT_HOST_PLATFORM,
|
||||
secretsFile,
|
||||
installationConfigFile,
|
||||
evidenceHostRegistryRoot: env.THT_EVIDENCE_HOST_REGISTRY_ROOT || undefined,
|
||||
modelCatalogFile,
|
||||
sensitivityNer,
|
||||
piAuthFile,
|
||||
|
||||
@@ -41,15 +41,17 @@ const runtimeModelSchema = z.object({
|
||||
supportsReasoningEffort: z.boolean(),
|
||||
supportsStore: z.boolean(),
|
||||
maxTokensField: z.string().optional(),
|
||||
thinkingFormat: z.enum(["qwen", "qwen-chat-template"]).optional(),
|
||||
}).strict().optional(),
|
||||
}).strict().optional(),
|
||||
}).strict().refine((session) => !session.compatibility?.thinkingFormat || session.reasoning, {
|
||||
message: "thinkingFormat requires reasoning: true",
|
||||
}).optional(),
|
||||
metadataGeneration: z.object({ disableThinking: z.boolean() }).strict().optional(),
|
||||
}).strict();
|
||||
|
||||
const catalogSchema = z.object({
|
||||
schemaVersion: z.literal(1),
|
||||
defaultSession: canonicalId,
|
||||
defaultMetadataGeneration: canonicalId.optional(),
|
||||
schemaVersion: z.literal(2),
|
||||
defaultInteraction: canonicalId,
|
||||
embedding: z.object({ id: canonicalId, dimensions: z.number().int().positive() }).strict(),
|
||||
models: z.array(runtimeModelSchema).max(64),
|
||||
}).strict();
|
||||
@@ -57,8 +59,7 @@ const catalogSchema = z.object({
|
||||
export type RuntimeModel = z.infer<typeof runtimeModelSchema>;
|
||||
|
||||
export interface RuntimeModelCatalog {
|
||||
readonly defaultSession: string | null;
|
||||
readonly defaultMetadataGeneration: string | null;
|
||||
readonly defaultInteraction: string | null;
|
||||
readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
|
||||
sessionModels(): readonly RuntimeModel[];
|
||||
metadataModels(): readonly RuntimeModel[];
|
||||
@@ -66,19 +67,21 @@ export interface RuntimeModelCatalog {
|
||||
}
|
||||
|
||||
class RestartLoadedRuntimeModelCatalog implements RuntimeModelCatalog {
|
||||
readonly defaultSession: string | null;
|
||||
readonly defaultMetadataGeneration: string | null;
|
||||
readonly defaultInteraction: string | null;
|
||||
readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
|
||||
readonly #sessions: readonly RuntimeModel[];
|
||||
readonly #metadata: readonly RuntimeModel[];
|
||||
readonly #sessionIds: ReadonlySet<string>;
|
||||
|
||||
constructor(catalog?: z.infer<typeof catalogSchema>) {
|
||||
this.defaultSession = catalog?.defaultSession ?? null;
|
||||
this.defaultMetadataGeneration = catalog?.defaultMetadataGeneration ?? null;
|
||||
this.defaultInteraction = catalog?.defaultInteraction ?? null;
|
||||
this.embedding = catalog ? Object.freeze({ ...catalog.embedding }) : null;
|
||||
this.#sessions = Object.freeze((catalog?.models ?? []).filter((model) => model.session !== undefined));
|
||||
this.#metadata = Object.freeze((catalog?.models ?? []).filter((model) => model.metadataGeneration !== undefined));
|
||||
// One operational list. Session-only installations can still run Core, but once Admin
|
||||
// LLM models are configured every selectable model must support both adapters.
|
||||
const hasMetadata = catalog?.models.some((model) => model.metadataGeneration !== undefined);
|
||||
this.#sessions = Object.freeze((catalog?.models ?? []).filter((model) =>
|
||||
model.session !== undefined && (!hasMetadata || model.metadataGeneration !== undefined)));
|
||||
this.#metadata = Object.freeze(this.#sessions.filter((model) => model.metadataGeneration !== undefined));
|
||||
this.#sessionIds = new Set(this.#sessions.map((model) => model.id));
|
||||
}
|
||||
|
||||
@@ -129,11 +132,9 @@ export function loadRuntimeModelCatalog(file?: string): RuntimeModelCatalog {
|
||||
if (ids.size !== parsed.data.models.length) throw new Error("runtime model catalog contains duplicate models");
|
||||
const sessions = parsed.data.models.filter((model) => model.session !== undefined).map((model) => model.id);
|
||||
const metadata = parsed.data.models.filter((model) => model.metadataGeneration !== undefined).map((model) => model.id);
|
||||
if (!sessions.includes(parsed.data.defaultSession)) throw new Error("runtime model catalog session default is invalid");
|
||||
if ((metadata.length > 0) !== (parsed.data.defaultMetadataGeneration !== undefined)
|
||||
|| (parsed.data.defaultMetadataGeneration !== undefined
|
||||
&& !metadata.includes(parsed.data.defaultMetadataGeneration))) {
|
||||
throw new Error("runtime model catalog metadata default is invalid");
|
||||
if (!sessions.includes(parsed.data.defaultInteraction)
|
||||
|| (metadata.length > 0 && !metadata.includes(parsed.data.defaultInteraction))) {
|
||||
throw new Error("runtime model catalog interaction default is invalid");
|
||||
}
|
||||
return new RestartLoadedRuntimeModelCatalog(parsed.data);
|
||||
}
|
||||
|
||||
@@ -16,12 +16,16 @@ import { loadSettings } from "./settings/settings-store.js";
|
||||
import { ThtRunner, type SessionRow } from "./tht/tht-runner.js";
|
||||
import { WorkspaceRegistry } from "./workspaces/registry.js";
|
||||
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
||||
import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js";
|
||||
import { resolveCatalogRuntimeBinding } from "./catalog/runtime-binding.js";
|
||||
import { CatalogService } from "./catalog/service.js";
|
||||
import { validateOperationalWorkspace } from "./workspaces/schema.js";
|
||||
import { createCatalogRepository } from "./catalog/repository.js";
|
||||
import type { CatalogRepository } from "./catalog/types.js";
|
||||
|
||||
type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status"
|
||||
| "session-inventory" | "workflow-doctor" | "workspace-integrity"
|
||||
| "pi-test" | "effective-settings";
|
||||
| "workspace-pull" | "workspace-test" | "pi-test" | "effective-settings";
|
||||
|
||||
const lifecyclePrincipal: PrincipalContext = {
|
||||
issuer: "tht-operator-command",
|
||||
@@ -119,6 +123,119 @@ async function workspaceIntegrity(config: AppConfig): Promise<{
|
||||
return { ready: true, ...integrity };
|
||||
}
|
||||
|
||||
async function workspacePull(config: AppConfig): Promise<{
|
||||
ready: boolean;
|
||||
status: "succeeded" | "degraded";
|
||||
branch: string;
|
||||
head?: string;
|
||||
degraded: boolean;
|
||||
}> {
|
||||
const status = await new WorkspaceRegistry(config.workspaceRegistry).pull();
|
||||
return {
|
||||
ready: !status.degraded,
|
||||
status: status.degraded ? "degraded" : "succeeded",
|
||||
branch: status.branch,
|
||||
...(status.head ? { head: status.head } : {}),
|
||||
degraded: status.degraded,
|
||||
};
|
||||
}
|
||||
|
||||
interface WorkspaceTestReport {
|
||||
id: string;
|
||||
status: "ready" | "failed";
|
||||
database: "reachable" | "not_configured" | "failed";
|
||||
diagnostics: string[];
|
||||
}
|
||||
|
||||
async function workspaceTest(config: AppConfig): Promise<{
|
||||
ready: boolean;
|
||||
workspaces: WorkspaceTestReport[];
|
||||
}> {
|
||||
if (!config.catalogDatabase) throw new Error("Catalog database is not configured");
|
||||
const registry = new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const revisions = await registry.list();
|
||||
const repository = createCatalogRepository(config.catalogDatabase);
|
||||
try {
|
||||
const secretStore = new WorkspaceSecretStore({
|
||||
root: config.workspaceSecretStoreRoot,
|
||||
runtimeRoot: config.workspaceSecretRuntimeRoot,
|
||||
installationId: config.workspaceRegistry.installationId,
|
||||
});
|
||||
const catalogService = new CatalogService(
|
||||
repository,
|
||||
registry,
|
||||
secretStore,
|
||||
config.workspaceRegistry.secretRoots,
|
||||
config.workspaceDiagnosticTimeoutMs,
|
||||
);
|
||||
const diagnose = createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
});
|
||||
const databases = await repository.list();
|
||||
const reports: WorkspaceTestReport[] = [];
|
||||
for (const revision of revisions) {
|
||||
const diagnostics: string[] = [];
|
||||
let workspace: ReturnType<typeof validateOperationalWorkspace>;
|
||||
try {
|
||||
workspace = validateOperationalWorkspace((await registry.read(revision.id)).workspace);
|
||||
} catch {
|
||||
reports.push({ id: revision.id, status: "failed", database: "not_configured", diagnostics: ["workspace_invalid"] });
|
||||
continue;
|
||||
}
|
||||
const database = databases.find((candidate) => candidate.workspaceId === revision.id);
|
||||
if (!database) {
|
||||
reports.push({ id: revision.id, status: "failed", database: "not_configured", diagnostics: ["database_binding_missing"] });
|
||||
continue;
|
||||
}
|
||||
let tested;
|
||||
try {
|
||||
tested = await catalogService.test(database);
|
||||
} catch {
|
||||
reports.push({ id: revision.id, status: "failed", database: "failed", diagnostics: ["database_unreachable"] });
|
||||
continue;
|
||||
}
|
||||
if (!tested || tested.connectionStatus !== "reachable") {
|
||||
reports.push({ id: revision.id, status: "failed", database: "failed", diagnostics: ["database_unreachable"] });
|
||||
continue;
|
||||
}
|
||||
let lease: ReturnType<typeof resolveCatalogRuntimeBinding>;
|
||||
try {
|
||||
lease = resolveCatalogRuntimeBinding({
|
||||
workspace,
|
||||
database: tested,
|
||||
environment: process.env,
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
secretStore,
|
||||
});
|
||||
} catch {
|
||||
reports.push({ id: revision.id, status: "failed", database: "reachable", diagnostics: ["binding_missing"] });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// CatalogService.test() above is the authoritative database probe and records its
|
||||
// outcome. The remaining diagnoser pass checks Evidence and internal semantic services;
|
||||
// skipping its legacy DWH probe avoids requiring a second response-shape contract for a
|
||||
// REST health endpoint.
|
||||
const result = await diagnose(lease.workspace, lease.bindings, { writeProbe: false, skipDwh: true });
|
||||
diagnostics.push(...result.diagnostics.map((diagnostic) => diagnostic.code));
|
||||
const ready = result.activatable;
|
||||
reports.push({ id: revision.id, status: ready ? "ready" : "failed", database: "reachable", diagnostics });
|
||||
} catch {
|
||||
reports.push({ id: revision.id, status: "failed", database: "reachable", diagnostics: ["connector_unavailable"] });
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
}
|
||||
return { ready: reports.length > 0 && reports.every((report) => report.status === "ready"), workspaces: reports };
|
||||
} finally {
|
||||
await repository.close?.();
|
||||
}
|
||||
}
|
||||
|
||||
export async function runOperatorAction(
|
||||
action: OperatorAction,
|
||||
config: AppConfig,
|
||||
@@ -132,6 +249,8 @@ export async function runOperatorAction(
|
||||
if (action === "session-inventory") return await sessionInventory(config);
|
||||
if (action === "workflow-doctor") return await workflowDiagnostics(config);
|
||||
if (action === "workspace-integrity") return await workspaceIntegrity(config);
|
||||
if (action === "workspace-pull") return await workspacePull(config);
|
||||
if (action === "workspace-test") return await workspaceTest(config);
|
||||
const modelCatalog = loadRuntimeModelCatalog(config.modelCatalogFile);
|
||||
if (action === "effective-settings") {
|
||||
return effectiveSettings(config, loadSettings(config), modelCatalog);
|
||||
@@ -146,6 +265,7 @@ async function main(): Promise<void> {
|
||||
if (!action || ![
|
||||
"maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory",
|
||||
"workflow-doctor", "workspace-integrity", "pi-test", "effective-settings",
|
||||
"workspace-pull", "workspace-test",
|
||||
].includes(action)) throw new Error("invalid operator action");
|
||||
const result = await runOperatorAction(action, loadConfig(process.env));
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
validateDeclarativePiConfig,
|
||||
} from "./managed-config.js";
|
||||
import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
|
||||
import { secretValue } from "../config/secret-bundle.js";
|
||||
|
||||
export interface PiModel {
|
||||
provider: string;
|
||||
@@ -64,6 +65,14 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModels
|
||||
}
|
||||
|
||||
const env = buildPiChildEnv({});
|
||||
// Pi's availability enumeration also needs the catalog-owned credentials for built-in
|
||||
// providers. It must keep working after their obsolete Pi auth entries are removed.
|
||||
for (const model of opts.modelCatalog?.sessionModels() ?? []) {
|
||||
const name = model.authentication.mode === "secret_env" ? model.authentication.apiKeyEnv : undefined;
|
||||
if (!name) continue;
|
||||
const value = secretValue(cfg, name);
|
||||
if (value) env[name] = value;
|
||||
}
|
||||
delete env.THT_DATA_ROOT;
|
||||
if (cfg.dataRoot !== undefined) env.THT_DATA_ROOT = cfg.dataRoot;
|
||||
const child = spawnFn(cfg.piBin, ["--mode", "rpc"], { cwd: cfg.harnessDir, env });
|
||||
|
||||
@@ -138,7 +138,7 @@ export interface PiRuntimeAgentSnapshot {
|
||||
* Bind a session Pi process to the exact managed auth/model bytes validated at spawn time.
|
||||
* Other agent resources remain live through symlinks, while session storage stays persistent.
|
||||
*/
|
||||
export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot {
|
||||
export function createPiRuntimeAgentSnapshot(options: { excludeAuthProvider?: string } = {}): PiRuntimeAgentSnapshot {
|
||||
const sourceAgentDir = configuredPiAgentDir();
|
||||
const auth = readPiAgentFile(sourceAgentDir, "auth.json", true);
|
||||
const models = readPiAgentFile(sourceAgentDir, "models.json", true);
|
||||
@@ -165,7 +165,18 @@ export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot {
|
||||
);
|
||||
}
|
||||
if (auth !== undefined) {
|
||||
writeFileSync(join(snapshotDir, "auth.json"), auth, { flag: "wx", mode: 0o600 });
|
||||
let effectiveAuth = auth;
|
||||
if (options.excludeAuthProvider) {
|
||||
const parsed = parsePiConfigJson(auth);
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new PiManagedConfigError();
|
||||
const provider = options.excludeAuthProvider.trim().toLowerCase();
|
||||
effectiveAuth = JSON.stringify(Object.fromEntries(
|
||||
Object.entries(parsed).filter(([key]) => key.trim().toLowerCase() !== provider),
|
||||
));
|
||||
}
|
||||
// secret_env is authoritative for this provider. Keep the operator's auth file intact,
|
||||
// but do not let an old Pi credential override the shared bundle inside this child.
|
||||
writeFileSync(join(snapshotDir, "auth.json"), effectiveAuth, { flag: "wx", mode: 0o600 });
|
||||
}
|
||||
if (models !== undefined) {
|
||||
writeFileSync(join(snapshotDir, "models.json"), models, { flag: "wx", mode: 0o600 });
|
||||
|
||||
@@ -36,6 +36,7 @@ export interface PiInstallationConfig {
|
||||
}
|
||||
|
||||
export interface PiStatus {
|
||||
hostPlatform: "linux" | "macos" | "windows";
|
||||
version?: string;
|
||||
ready: boolean;
|
||||
credentials: PiCredentialStatus;
|
||||
@@ -92,6 +93,9 @@ interface PiManagementDeps {
|
||||
}
|
||||
|
||||
export function createPiManagement(config: AppConfig, deps: PiManagementDeps): PiManagementService {
|
||||
const platform = config.hostPlatform ?? process.platform;
|
||||
const hostPlatform = platform === "darwin" ? "macos"
|
||||
: platform === "windows" || platform === "win32" ? "windows" : "linux";
|
||||
const now = deps.now ?? (() => new Date());
|
||||
const diagnostics: string[] = [];
|
||||
const addDiagnostic = (message: string): void => {
|
||||
@@ -106,23 +110,23 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
||||
});
|
||||
const credentialStatus = deps.credentialStatus ?? ((provider: string | undefined) => {
|
||||
try {
|
||||
const model = deps.modelCatalog.defaultSession
|
||||
? deps.modelCatalog.sessionModels().find((entry) => entry.id === deps.modelCatalog.defaultSession)
|
||||
const model = deps.modelCatalog.defaultInteraction
|
||||
? deps.modelCatalog.sessionModels().find((entry) => entry.id === deps.modelCatalog.defaultInteraction)
|
||||
: undefined;
|
||||
const credentialName = model?.authentication.mode === "secret_env"
|
||||
? model.authentication.apiKeyEnv
|
||||
: undefined;
|
||||
const configuredApiKey = configuredPiProviderApiKey(
|
||||
const configuredApiKey = credentialName ? `$${credentialName}` : configuredPiProviderApiKey(
|
||||
readConfiguredPiAgentFile("models.json", true),
|
||||
provider,
|
||||
) ?? (credentialName ? `$${credentialName}` : undefined);
|
||||
);
|
||||
return piProviderCredentialStatus({
|
||||
provider,
|
||||
authProviders: loadPiAuthProviders(),
|
||||
authProviders: credentialName ? new Set() : loadPiAuthProviders(),
|
||||
resolveCredentialValue: () => credentialName
|
||||
? secretValue(config, credentialName)
|
||||
: config.modelCatalogFile ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
|
||||
credentialFile: config.modelApiKeyFile,
|
||||
credentialFile: credentialName ? undefined : config.modelApiKeyFile,
|
||||
configuredApiKey,
|
||||
});
|
||||
} catch {
|
||||
@@ -152,8 +156,8 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
||||
const installationConfig = (): PiInstallationConfig => {
|
||||
const settings = readSettings();
|
||||
const reasoning = config.defaults.thinking ?? settings.thinking;
|
||||
const selected = deps.modelCatalog.defaultSession
|
||||
? splitCanonicalModelId(deps.modelCatalog.defaultSession)
|
||||
const selected = deps.modelCatalog.defaultInteraction
|
||||
? splitCanonicalModelId(deps.modelCatalog.defaultInteraction)
|
||||
: undefined;
|
||||
return {
|
||||
...(selected ? selected : {}),
|
||||
@@ -169,11 +173,11 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
||||
try {
|
||||
const currentVersion = await version();
|
||||
addDiagnostic("Pi version probe succeeded");
|
||||
return { version: currentVersion, ready: true, credentials, config: current, checkedAt };
|
||||
return { hostPlatform, version: currentVersion, ready: true, credentials, config: current, checkedAt };
|
||||
} catch (error) {
|
||||
const message = stableMessage(error, "Pi runtime is unavailable");
|
||||
addDiagnostic(message);
|
||||
return { ready: false, credentials, config: current, checkedAt, message };
|
||||
return { hostPlatform, ready: false, credentials, config: current, checkedAt, message };
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ export interface SessionRuntime {
|
||||
}
|
||||
|
||||
export interface RuntimeOptions {
|
||||
interactionLanguage?: string | null;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
thinking?: string;
|
||||
@@ -48,6 +49,7 @@ export class PiProcessManager {
|
||||
private spawnFn: (
|
||||
sessionId: string, author: string, provider: string | undefined, model: string | undefined,
|
||||
principal?: PrincipalContext, runtimeConfigPath?: string,
|
||||
interactionLanguage?: string | null,
|
||||
) => ChildProcessWithoutNullStreams;
|
||||
private loadAuthProviders: (agentDir: string) => ReadonlySet<string>;
|
||||
private modelCatalog: RuntimeModelCatalog;
|
||||
@@ -63,15 +65,15 @@ export class PiProcessManager {
|
||||
) {
|
||||
this.modelCatalog = opts?.modelCatalog ?? loadRuntimeModelCatalog(cfg.modelCatalogFile);
|
||||
this.modelCatalogConfigured = cfg.modelCatalogFile !== undefined
|
||||
|| this.modelCatalog.defaultSession !== null;
|
||||
|| this.modelCatalog.defaultInteraction !== null;
|
||||
this.loadAuthProviders = opts?.authProviders
|
||||
?? ((agentDir) => loadPiAuthProviders({ agentDir }));
|
||||
if (opts?.spawnFn) {
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
|
||||
this.spawnPi(opts.spawnFn!, sessionId, author, provider, model, principal, runtimeConfigPath);
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath, language) =>
|
||||
this.spawnPi(opts.spawnFn!, sessionId, author, provider, model, principal, runtimeConfigPath, language);
|
||||
} else {
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
|
||||
this.spawnPi(nodeSpawn, sessionId, author, provider, model, principal, runtimeConfigPath);
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath, language) =>
|
||||
this.spawnPi(nodeSpawn, sessionId, author, provider, model, principal, runtimeConfigPath, language);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,33 +87,35 @@ export class PiProcessManager {
|
||||
private spawnPi(
|
||||
spawnFn: SpawnFn, sessionId: string, author: string, provider: string | undefined,
|
||||
model: string | undefined, principal?: PrincipalContext, runtimeConfigPath?: string,
|
||||
interactionLanguage?: string | null,
|
||||
): ChildProcessWithoutNullStreams {
|
||||
// This is the final shared boundary for createFor(), spawnFor(), and resume(). Validate
|
||||
// before auth-provider inspection, then make Pi consume the exact copied bytes rather than
|
||||
// reopening mutable mounted auth/models files after this check.
|
||||
const agent = createPiRuntimeAgentSnapshot();
|
||||
const catalogModel = provider && model
|
||||
? this.modelCatalog.sessionModels().find((entry) => entry.provider === provider && entry.model === model)
|
||||
: undefined;
|
||||
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
||||
? catalogModel.authentication.apiKeyEnv : undefined;
|
||||
const agent = createPiRuntimeAgentSnapshot({ excludeAuthProvider: credentialName ? provider : undefined });
|
||||
let child: ChildProcessWithoutNullStreams | undefined;
|
||||
try {
|
||||
const catalogModel = provider && model
|
||||
? this.modelCatalog.sessionModels()
|
||||
.find((entry) => entry.provider === provider && entry.model === model)
|
||||
: undefined;
|
||||
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
||||
? catalogModel.authentication.apiKeyEnv
|
||||
: undefined;
|
||||
const projectedApiKey = configuredPiProviderApiKey(agent.models, provider)
|
||||
?? (credentialName ? `$${credentialName}` : undefined);
|
||||
const projectedApiKey = credentialName ? `$${credentialName}`
|
||||
: configuredPiProviderApiKey(agent.models, provider);
|
||||
const env = buildPiChildEnv({
|
||||
provider,
|
||||
authProviders: this.loadAuthProviders(agent.agentDir),
|
||||
authProviders: credentialName ? new Set() : this.loadAuthProviders(agent.agentDir),
|
||||
credentialValue: credentialName
|
||||
? secretValue(this.cfg, credentialName)
|
||||
: this.modelCatalogConfigured ? undefined : secretValue(this.cfg, "THT_MODEL_API_KEY"),
|
||||
credentialFile: this.cfg.modelApiKeyFile,
|
||||
credentialFile: credentialName ? undefined : this.cfg.modelApiKeyFile,
|
||||
configuredApiKey: projectedApiKey,
|
||||
additions: { THT_SESSION: sessionId, THT_AUTHOR: author },
|
||||
});
|
||||
env.PI_CODING_AGENT_DIR = agent.agentDir;
|
||||
// A launch hint only: the gate reads the authoritative manifest before each turn.
|
||||
delete env.THT_INTERACTION_LANGUAGE;
|
||||
if (interactionLanguage) env.THT_INTERACTION_LANGUAGE = interactionLanguage;
|
||||
env.PI_CODING_AGENT_SESSION_DIR = agent.sessionDir;
|
||||
clearPrincipalEnvironment(env);
|
||||
if (principal) Object.assign(env, principalEnvironment(principal));
|
||||
@@ -189,7 +193,9 @@ export class PiProcessManager {
|
||||
const model = o.model ?? this.cfg.defaults.model;
|
||||
let child: ChildProcessWithoutNullStreams;
|
||||
try {
|
||||
child = this.spawnFn(sessionId, author, provider, model, o.principal, o.runtimeConfig?.path);
|
||||
child = this.spawnFn(
|
||||
sessionId, author, provider, model, o.principal, o.runtimeConfig?.path, o.interactionLanguage,
|
||||
);
|
||||
} catch (error) {
|
||||
o.runtimeConfig?.release();
|
||||
throw error;
|
||||
@@ -298,8 +304,13 @@ export class PiProcessManager {
|
||||
}
|
||||
|
||||
async resume(sessionId: string, tht: ThtRunner): Promise<SessionRuntime> {
|
||||
const manifest = await tht.sessionShow(sessionId) as { provider?: string; model?: string; thinking?: string } | null;
|
||||
const manifest = await tht.sessionShow(sessionId) as {
|
||||
provider?: string; model?: string; thinking?: string; interaction_language?: string | null;
|
||||
} | null;
|
||||
const language = manifest?.interaction_language
|
||||
?? (await tht.ensureInteractionLanguage(sessionId)).interaction_language;
|
||||
return this.spawnFor(sessionId, {
|
||||
interactionLanguage: language,
|
||||
provider: manifest?.provider,
|
||||
model: manifest?.model,
|
||||
thinking: manifest?.thinking,
|
||||
|
||||
@@ -70,28 +70,29 @@ export function createPiProviderSmoke(
|
||||
try {
|
||||
const canonicalProvider = canonicalPiProvider(provider);
|
||||
if (!canonicalProvider || timeoutMs <= 0) throw providerFailure();
|
||||
const configuredAuthProviders = authProviders();
|
||||
const configuredAuthProviders = new Set(authProviders());
|
||||
const configuredModels = options.readModelsStore
|
||||
? options.readModelsStore()
|
||||
: readConfiguredPiAgentFile("models.json", true);
|
||||
const catalog = options.modelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
|
||||
const catalogConfigured = config.modelCatalogFile !== undefined
|
||||
|| catalog.defaultSession !== null;
|
||||
|| catalog.defaultInteraction !== null;
|
||||
const catalogModel = catalog.sessionModels()
|
||||
.find((entry) => entry.provider === canonicalProvider && entry.model === model);
|
||||
const upstreamModel = catalogModel?.upstreamModel ?? model;
|
||||
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
||||
? catalogModel.authentication.apiKeyEnv
|
||||
: undefined;
|
||||
const projectedApiKey = configuredPiProviderApiKey(configuredModels, canonicalProvider)
|
||||
?? (credentialName ? `$${credentialName}` : undefined);
|
||||
if (credentialName) configuredAuthProviders.delete(canonicalProvider);
|
||||
const projectedApiKey = credentialName ? `$${credentialName}`
|
||||
: configuredPiProviderApiKey(configuredModels, canonicalProvider);
|
||||
const env = buildPiChildEnv({
|
||||
provider: canonicalProvider,
|
||||
authProviders: configuredAuthProviders,
|
||||
credentialValue: credentialName
|
||||
? secretValue(config, credentialName)
|
||||
: catalogConfigured ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
|
||||
credentialFile: config.modelApiKeyFile,
|
||||
credentialFile: credentialName ? undefined : config.modelApiKeyFile,
|
||||
configuredApiKey: projectedApiKey,
|
||||
});
|
||||
clearPrincipalEnvironment(env);
|
||||
|
||||
@@ -1,60 +1,19 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
|
||||
import { databaseConfigurationSchema as configSchema } from "../catalog/configuration-schema.js";
|
||||
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
|
||||
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
DATABASE_TRANSPORTS,
|
||||
type CatalogRepository,
|
||||
type DatabaseConfigurationInput,
|
||||
} from "../catalog/types.js";
|
||||
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
|
||||
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
|
||||
const nonEmpty = z.string().trim().min(1).max(512);
|
||||
const port = z.number().int().min(1).max(65_535);
|
||||
const optionalText = nonEmpty.optional();
|
||||
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
|
||||
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
|
||||
const bindingSchema = z.object({
|
||||
transport: z.enum(DATABASE_TRANSPORTS),
|
||||
host: optionalText,
|
||||
port: port.optional(),
|
||||
username: optionalText,
|
||||
baseUrl: z.string().max(2048)
|
||||
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
|
||||
.optional(),
|
||||
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
|
||||
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
|
||||
tlsServername: optionalText,
|
||||
sshHost,
|
||||
sshPort: port.optional(),
|
||||
sshUsername,
|
||||
sshTargetHost: sshHost,
|
||||
sshTargetPort: port.optional(),
|
||||
}).strict().superRefine((binding, context) => {
|
||||
const required = binding.transport === "postgres_direct"
|
||||
? ["host", "port", "username"] as const
|
||||
: binding.transport === "rest_api"
|
||||
? ["baseUrl", "restPath", "restAuth"] as const
|
||||
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
|
||||
for (const field of required) {
|
||||
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
|
||||
}
|
||||
});
|
||||
const configSchema = z.object({
|
||||
workspaceId: workspaceIdSchema,
|
||||
engine: z.literal("postgres"),
|
||||
databaseName: identifier,
|
||||
schema: identifier,
|
||||
binding: bindingSchema,
|
||||
}).strict();
|
||||
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
|
||||
const secretNames = [
|
||||
"password",
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import path from "node:path";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { requirePermission, isPrincipalContext } from "../auth/authorization.js";
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
|
||||
|
||||
const params = z.object({ workspaceId: z.string().regex(/^[a-z][a-z0-9-]{2,62}$/),
|
||||
evidenceId: z.string().regex(/^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$/).optional() });
|
||||
const query = z.object({
|
||||
q: z.string().max(1000).optional(), kind: z.enum(["domain", "glossary", "enum", "example", "mapping", "normalization", "formula", "reference"]).optional(),
|
||||
purpose: z.enum(["disambiguation", "rewriting", "schema_linking", "sql_generation"]).optional(),
|
||||
status: z.enum(["new", "modified", "active", "removed", "review_required", "legacy", "invalid"]).optional(),
|
||||
concept: z.string().max(300).optional(), table: z.string().max(300).optional(), column: z.string().max(300).optional(),
|
||||
source: z.string().max(300).optional(), language: z.string().max(30).optional(),
|
||||
sort: z.enum(["title", "id", "kind", "status"]).optional(), direction: z.enum(["asc", "desc"]).optional(),
|
||||
page: z.coerce.number().int().positive().optional(), page_size: z.coerce.number().int().min(1).max(100).optional(),
|
||||
}).strict();
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
||||
|
||||
export function evidenceRoutes(app: FastifyInstance, deps: {
|
||||
runner: Pick<ThtRunner, "withPrincipal">; registry: Pick<WorkspaceRegistry, "list">;
|
||||
registryRoot: string; hostRegistryRoot?: string;
|
||||
service: Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
|
||||
}) {
|
||||
app.post("/workspaces/:workspaceId/evidence/sources", async (request, reply) => {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId } = params.parse(request.params);
|
||||
const body = z.discriminatedUnion("action", [
|
||||
z.object({ action: z.literal("refresh") }).strict(),
|
||||
z.object({ action: z.literal("decide"), sourceId: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
revision: z.string().regex(/^[a-f0-9]{64}$/), decision: z.enum(["keep", "replace"]) }).strict(),
|
||||
]).parse(request.body);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
if (!deps.service.evidenceSources) throw new Error("Evidence service unavailable");
|
||||
return await deps.service.evidenceSources({ workspaceId, ...body, actor: principal.subject });
|
||||
} catch (error) {
|
||||
return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
|
||||
message: error instanceof z.ZodError ? "Invalid source request." : "Evidence source service is unavailable." });
|
||||
}
|
||||
});
|
||||
app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence", read);
|
||||
app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence/:evidenceId", read);
|
||||
async function read(request: import("fastify").FastifyRequest, reply: import("fastify").FastifyReply) {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId, evidenceId } = params.parse(request.params);
|
||||
const filters = query.parse(request.query);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
const root = path.join(deps.registryRoot, "repo", workspaceId);
|
||||
const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
|
||||
["evidence", "admin", "--workspace", workspaceId],
|
||||
JSON.stringify({ root, query: { ...filters, ...(evidenceId ? { id: evidenceId } : {}) } }),
|
||||
);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.code !== 0) return reply.code(503).send(payload);
|
||||
if (evidenceId && !payload.item) return reply.code(404).send({ code: "evidence_not_found", message: "Evidence was not found." });
|
||||
const host = deps.hostRegistryRoot;
|
||||
const hostPath = host && (path.isAbsolute(host) || path.win32.isAbsolute(host))
|
||||
? (path.win32.isAbsolute(host) && !path.isAbsolute(host) ? path.win32 : path).join(host, "repo") : null;
|
||||
const hostJoin = hostPath && path.win32.isAbsolute(hostPath) && !path.isAbsolute(hostPath) ? path.win32.join : path.join;
|
||||
return { ...payload, location: { repository: hostPath, workspace: hostPath ? hostJoin(hostPath, workspaceId) : null,
|
||||
runtime_workspace: root, host: "Installation host", command: `tht workspace evidence consolidate --workspace ${workspaceId}`,
|
||||
git_commands: hostPath ? [
|
||||
`git -C ${quote(hostPath)} status --short -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} diff -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} add -A -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} commit --only -m 'Curate Evidence' -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} push`,
|
||||
] : [] } };
|
||||
} catch (error) {
|
||||
return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
|
||||
message: error instanceof z.ZodError ? "Evidence filters are invalid." : "Evidence archive is unavailable." });
|
||||
}
|
||||
}
|
||||
app.post("/workspaces/:workspaceId/evidence/consolidate", async (request, reply) => {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId } = params.parse(request.params);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
if (!deps.service.consolidateEvidence) throw new Error("Evidence service unavailable");
|
||||
return await deps.service.consolidateEvidence({ workspaceId });
|
||||
} catch { return reply.code(503).send({ code: "evidence_unavailable", message: "Evidence consolidation is unavailable." }); }
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
|
||||
|
||||
const paramsSchema = z.object({
|
||||
workspaceId: z.string().regex(/^[a-z][a-z0-9_-]{0,63}$/),
|
||||
cardId: z.string().regex(/^mem-[0-9a-f-]{36}$/).optional(),
|
||||
});
|
||||
const family = z.enum(["domain_clarification", "sql_rule", "solved_question", "explained_error"]);
|
||||
const cardSchema = z.object({
|
||||
family, subject: z.string().trim().min(1).max(1000),
|
||||
detail: z.string().max(50000).default(""), scope: z.string().trim().min(1).max(10000),
|
||||
rationale: z.string().max(10000).default(""), question: z.string().max(10000).default(""),
|
||||
sql: z.string().max(100000).default(""),
|
||||
concepts: z.array(z.string().trim().min(1).max(200)).max(100).default([]),
|
||||
dependencies: z.array(z.object({
|
||||
database: z.string().trim().min(1).max(200), schema_name: z.string().max(200).default(""),
|
||||
table: z.string().max(200).default(""), column: z.string().max(200).default(""),
|
||||
}).strict()).max(200).default([]),
|
||||
links: z.array(z.object({
|
||||
target_id: z.string().min(1).max(100), meaning: z.string().trim().min(1).max(1000),
|
||||
}).strict()).max(200).default([]),
|
||||
}).strict();
|
||||
const querySchema = z.object({
|
||||
q: z.string().max(1000).optional(), family: family.optional(),
|
||||
concept: z.string().max(200).optional(), database: z.string().max(200).optional(),
|
||||
table: z.string().max(200).optional(), column: z.string().max(200).optional(),
|
||||
origin: z.enum(["manual", "workflow"]).optional(),
|
||||
updated_after: z.iso.datetime({ offset: true }).optional(),
|
||||
updated_before: z.iso.datetime({ offset: true }).optional(),
|
||||
page: z.coerce.number().int().positive().optional(),
|
||||
page_size: z.coerce.number().int().min(1).max(100).optional(),
|
||||
sort: z.enum(["updated_at", "created_at", "subject", "family"]).optional(),
|
||||
direction: z.enum(["asc", "desc"]).optional(),
|
||||
}).strict();
|
||||
|
||||
export function memoryRoutes(app: FastifyInstance, deps: {
|
||||
runner: Pick<ThtRunner, "withPrincipal">;
|
||||
registry: Pick<WorkspaceRegistry, "list" | "read">;
|
||||
runtime: SemanticRuntimeConfig;
|
||||
}) {
|
||||
const invoke = (action: string) => async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const principal = requirePermission(request, reply, "memory.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId, cardId } = paramsSchema.parse(request.params);
|
||||
const input = action === "list" ? querySchema.parse(request.query)
|
||||
: action === "create" || action === "update"
|
||||
? { card: cardSchema.parse(request.body), ...(cardId ? { id: cardId } : {}) }
|
||||
: cardId ? { id: cardId } : {};
|
||||
// Registry identity is enough: administrative browsing must not require a DWH binding.
|
||||
const workspaces = await deps.registry.list();
|
||||
if (!workspaces.some(workspace => workspace.id === workspaceId)) {
|
||||
return reply.code(404).send({ code: "workspace_invalid", message: "Workspace was not found." });
|
||||
}
|
||||
const { workspace } = await deps.registry.read(workspaceId);
|
||||
const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
|
||||
["memory", "admin", "--workspace", workspaceId],
|
||||
JSON.stringify({ action, request: input, runtime: {
|
||||
...deps.runtime, memoryLanguage: workspace.workspace.language,
|
||||
} }),
|
||||
);
|
||||
let payload;
|
||||
try { payload = JSON.parse(result.stdout); } catch {
|
||||
return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
|
||||
}
|
||||
if (result.code !== 0) {
|
||||
const codes: Record<string, number> = {
|
||||
memory_invalid: 400, memory_forbidden: 403, memory_not_found: 404,
|
||||
memory_conflict: 409, memory_unavailable: 503,
|
||||
};
|
||||
const code = typeof payload?.code === "string" && payload.code in codes
|
||||
? payload.code : "memory_unavailable";
|
||||
return reply.code(codes[code]).send({ code, message: "Memory operation could not be completed." });
|
||||
}
|
||||
return reply.code(action === "create" ? 201 : 200).send(payload);
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({ code: "memory_invalid", message: "Memory request is invalid." });
|
||||
}
|
||||
return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
|
||||
}
|
||||
};
|
||||
app.get("/workspaces/:workspaceId/memory", invoke("list"));
|
||||
app.post("/workspaces/:workspaceId/memory", invoke("create"));
|
||||
app.get("/workspaces/:workspaceId/memory/pending", invoke("pending"));
|
||||
app.get("/workspaces/:workspaceId/memory/:cardId", invoke("show"));
|
||||
app.put("/workspaces/:workspaceId/memory/:cardId", invoke("update"));
|
||||
app.delete("/workspaces/:workspaceId/memory/:cardId", invoke("delete"));
|
||||
app.post("/workspaces/:workspaceId/memory/:cardId/retry", invoke("retry"));
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
|
||||
import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import { splitCanonicalModelId, type RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
|
||||
import type { CatalogRepository } from "../catalog/types.js";
|
||||
import { interactionLanguage } from "../tht/interaction-language.js";
|
||||
|
||||
const BOOTSTRAP_FAILURE_MESSAGE =
|
||||
"Session startup failed. Check configuration and connectivity, then Resume the session.";
|
||||
@@ -381,8 +382,17 @@ export function sessionRoutes(
|
||||
app.post("/sessions", async (req, reply) => {
|
||||
const b = req.body as {
|
||||
question: string; name?: string; workspace?: string; workspaceId?: string;
|
||||
provider?: string; model?: string; thinking?: string;
|
||||
provider?: string; model?: string; thinking?: string; interactionLanguage?: unknown;
|
||||
};
|
||||
const language = interactionLanguage(b?.interactionLanguage);
|
||||
if (!language) return reply.code(400).send({
|
||||
code: "invalid_interaction_language",
|
||||
error: "interactionLanguage must be a well-formed BCP-47 language tag",
|
||||
});
|
||||
if ((b.provider === undefined) !== (b.model === undefined)
|
||||
|| (b.provider !== undefined && (typeof b.provider !== "string" || typeof b.model !== "string" || !b.provider || !b.model))) {
|
||||
return reply.code(400).send({ error: "provider and model must be supplied together" });
|
||||
}
|
||||
const principal = getPrincipal(req);
|
||||
let s: Settings;
|
||||
try { s = await d.getSettings(principal); } catch { return storageFailure(reply); }
|
||||
@@ -446,11 +456,9 @@ export function sessionRoutes(
|
||||
}
|
||||
}
|
||||
const requestedCanonical = b.provider && b.model ? `${b.provider}/${b.model}` : undefined;
|
||||
let selectedCanonical = requestedCanonical ?? d.modelCatalog.defaultSession;
|
||||
let modelWarning: string | undefined;
|
||||
if (selectedCanonical && d.modelCatalog.defaultSession && !d.modelCatalog.hasSession(selectedCanonical)) {
|
||||
selectedCanonical = d.modelCatalog.defaultSession;
|
||||
modelWarning = `Configured model ${requestedCanonical ?? "selection"} is unavailable; using ${selectedCanonical}.`;
|
||||
const selectedCanonical = requestedCanonical ?? d.modelCatalog.defaultInteraction;
|
||||
if (selectedCanonical && d.modelCatalog.defaultInteraction && !d.modelCatalog.hasSession(selectedCanonical)) {
|
||||
return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
|
||||
}
|
||||
const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : undefined;
|
||||
const provider = selected?.provider ?? b.provider;
|
||||
@@ -501,11 +509,15 @@ export function sessionRoutes(
|
||||
// registry snapshot. The legacy fallback stays available for sessions created before
|
||||
// the browser-local preference migration.
|
||||
let id: string;
|
||||
let sessionLanguage = language;
|
||||
try {
|
||||
({ id } = await runner.sessionNew({
|
||||
const created = await runner.sessionNew({
|
||||
question: b.question, name: b.name, workspaceConfigPath,
|
||||
workspaceId, workspaceRevision, provider, model, thinking,
|
||||
}));
|
||||
interactionLanguage: language,
|
||||
});
|
||||
id = created.id;
|
||||
sessionLanguage = created.interaction_language ?? language;
|
||||
manifestPersisted = true;
|
||||
if (revisionLease) {
|
||||
await revisionLease.markPersisted().catch((error: unknown) => {
|
||||
@@ -518,6 +530,7 @@ export function sessionRoutes(
|
||||
} catch { return storageFailure(reply); }
|
||||
const options = {
|
||||
provider, model, thinking,
|
||||
interactionLanguage: sessionLanguage,
|
||||
author: principal.displayName ?? principal.subject,
|
||||
principal,
|
||||
question: b.question,
|
||||
@@ -554,7 +567,7 @@ export function sessionRoutes(
|
||||
),
|
||||
() => d.mgr.start(id, rt, runtimeOptions),
|
||||
);
|
||||
return { id, ...(modelWarning ? { warning: modelWarning } : {}) };
|
||||
return { id };
|
||||
} finally {
|
||||
if (revisionLease && !manifestPersisted) {
|
||||
await revisionLease.abort().catch((error: unknown) => {
|
||||
@@ -626,6 +639,23 @@ export function sessionRoutes(
|
||||
} catch (error) { return lifecycleFailure(reply, error); }
|
||||
const rt = d.mgr.get(id);
|
||||
if (!rt) return reply.code(404).send({ error: "sessione non attiva" });
|
||||
const pending = rt.bridge.pendingWidget?.() as any;
|
||||
const response = (req.body as any)?.ui_response;
|
||||
if (pending?.widget === "archive-repair" && response?.id === pending.id && !response.control) {
|
||||
const choices = response.choices;
|
||||
if (!Array.isArray(choices) || choices.length !== 1)
|
||||
return reply.code(400).send({ error: "Select one archive repair choice" });
|
||||
if (choices[0] !== "continue" && choices[0] !== "reject") {
|
||||
const option = pending.repair?.options?.find((item: any) => item.id === choices[0]);
|
||||
if (!option || !["memory", "evidence"].includes(option.archive))
|
||||
return reply.code(400).send({ error: "Unknown archive repair choice" });
|
||||
const permission = option.archive === "memory" ? "memory.manage" : "evidence.manage";
|
||||
if (!principal.isAdmin || !hasPermission(principal, permission))
|
||||
return reply.code(403).send({ error: "Archive corrections require an administrator" });
|
||||
if (rt.ownerKey !== `${principal.issuer}\0${principal.subject}`)
|
||||
return reply.code(409).send({ error: "Resume the session with your account before correcting an archive" });
|
||||
}
|
||||
}
|
||||
if (!rt.bridge.respond((req.body as any).ui_response)) {
|
||||
return reply.code(409).send({ error: "risposta non corrispondente al gate in attesa" });
|
||||
}
|
||||
@@ -645,6 +675,13 @@ export function sessionRoutes(
|
||||
app.post("/sessions/:id/resume", async (req, reply) => {
|
||||
const id = (req.params as any).id;
|
||||
const principal = getPrincipal(req);
|
||||
if (Object.hasOwn(req.body ?? {}, "interactionLanguage")
|
||||
|| Object.hasOwn(req.body ?? {}, "interaction_language")) {
|
||||
return reply.code(400).send({
|
||||
code: "interaction_language_pinned",
|
||||
error: "Resume uses the session's persisted interaction language; overrides are not accepted",
|
||||
});
|
||||
}
|
||||
return withSessionLifecycle(id, async () => {
|
||||
let settings: Settings;
|
||||
let located: LocatedSession | undefined;
|
||||
@@ -662,11 +699,19 @@ export function sessionRoutes(
|
||||
const saved = manifest as {
|
||||
provider?: string; model?: string; thinking?: string;
|
||||
workspace_id?: string; workspace_revision?: string;
|
||||
interaction_language?: string | null;
|
||||
};
|
||||
const savedCanonical = saved.provider && saved.model ? `${saved.provider}/${saved.model}` : "";
|
||||
if (d.modelCatalog.defaultSession && (!savedCanonical || !d.modelCatalog.hasSession(savedCanonical))) {
|
||||
const requested = (req.body ?? {}) as { provider?: string; model?: string; thinking?: string };
|
||||
if ((requested.provider === undefined) !== (requested.model === undefined)
|
||||
|| (requested.provider !== undefined && (typeof requested.provider !== "string" || typeof requested.model !== "string" || !requested.provider || !requested.model))) {
|
||||
return reply.code(400).send({ error: "provider and model must be supplied together" });
|
||||
}
|
||||
const selectedCanonical = requested.provider && requested.model
|
||||
? `${requested.provider}/${requested.model}` : d.modelCatalog.defaultInteraction;
|
||||
if (d.modelCatalog.defaultInteraction && (!selectedCanonical || !d.modelCatalog.hasSession(selectedCanonical))) {
|
||||
return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
|
||||
}
|
||||
const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : saved;
|
||||
let workspaceConfigPath: string;
|
||||
let workspaceDescriptor: WorkspaceDescriptor | undefined;
|
||||
try {
|
||||
@@ -693,13 +738,19 @@ export function sessionRoutes(
|
||||
});
|
||||
}
|
||||
try { settings = await d.getSettings(principal); } catch { return storageFailure(reply); }
|
||||
let language = saved.interaction_language;
|
||||
if (language == null) {
|
||||
try {
|
||||
language = (await runner.ensureInteractionLanguage(id, workspaceConfigPath)).interaction_language;
|
||||
} catch { return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE }); }
|
||||
}
|
||||
// This check belongs inside the per-session lock: a preceding cold Resume may have
|
||||
// installed a running runtime while this request was waiting.
|
||||
const existing = d.mgr.get(id);
|
||||
if (existing) {
|
||||
const state = existing.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
return reply.code(200).send({ id, alreadyActive: true, workspaceId: saved.workspace_id });
|
||||
}
|
||||
}
|
||||
const ensure = await d.readiness.ensure(
|
||||
@@ -710,8 +761,9 @@ export function sessionRoutes(
|
||||
...(ensure.code ? { code: ensure.code } : {}),
|
||||
});
|
||||
const options = {
|
||||
provider: saved?.provider,
|
||||
model: saved?.model,
|
||||
provider: selected.provider,
|
||||
interactionLanguage: language,
|
||||
model: selected.model,
|
||||
thinking: saved?.thinking ?? settings.thinking,
|
||||
author: principal.displayName ?? principal.subject,
|
||||
principal,
|
||||
@@ -733,7 +785,7 @@ export function sessionRoutes(
|
||||
if (current) {
|
||||
const state = current.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
return reply.code(200).send({ id, alreadyActive: true, workspaceId: saved.workspace_id });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -774,7 +826,7 @@ export function sessionRoutes(
|
||||
d.mgr.configure(rt, runtimeOptions), null,
|
||||
() => d.mgr.start(id, rt, runtimeOptions),
|
||||
);
|
||||
return reply.code(200).send({ id, alreadyActive: false });
|
||||
return reply.code(200).send({ id, alreadyActive: false, workspaceId: saved.workspace_id });
|
||||
});
|
||||
});
|
||||
app.post("/sessions/:id/close", async (req, reply) => {
|
||||
|
||||
@@ -16,8 +16,8 @@ export function effectiveSettings(
|
||||
modelCatalog?: RuntimeModelCatalog,
|
||||
): Settings {
|
||||
const workspaces = listWorkspaces(cfg.harnessDir);
|
||||
const selected = modelCatalog?.defaultSession
|
||||
? splitCanonicalModelId(modelCatalog.defaultSession)
|
||||
const selected = modelCatalog?.defaultInteraction
|
||||
? splitCanonicalModelId(modelCatalog.defaultInteraction)
|
||||
: undefined;
|
||||
return {
|
||||
workspace: stored.workspace ?? workspaces[0]?.name,
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
/** Validate/canonicalize the tag; available translation catalogs belong to the UI. */
|
||||
export function interactionLanguage(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
try {
|
||||
const [canonical] = Intl.getCanonicalLocales(value);
|
||||
return canonical;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -59,6 +59,7 @@ export interface SessionRow {
|
||||
author: string | null;
|
||||
workspace_id?: string | null;
|
||||
workspace_revision?: string | null;
|
||||
interaction_language?: string | null;
|
||||
archived?: boolean;
|
||||
}
|
||||
|
||||
@@ -482,6 +483,7 @@ export class ThtRunner {
|
||||
|
||||
async sessionNew(o: {
|
||||
question: string;
|
||||
interactionLanguage?: string;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
thinking?: string;
|
||||
@@ -497,6 +499,7 @@ export class ThtRunner {
|
||||
["--provider", o.provider],
|
||||
["--model", o.model],
|
||||
["--thinking", o.thinking],
|
||||
["--interaction-language", o.interactionLanguage],
|
||||
["--name", o.name],
|
||||
["--workspace-id", o.workspaceId],
|
||||
["--workspace-revision", o.workspaceRevision],
|
||||
@@ -504,7 +507,7 @@ export class ThtRunner {
|
||||
if (v) a.push(f, v);
|
||||
}
|
||||
a.push("--json");
|
||||
return this.json<{ id: string }>(a, o.workspaceConfigPath ?? o.workspace);
|
||||
return this.json<{ id: string; interaction_language?: string }>(a, o.workspaceConfigPath ?? o.workspace);
|
||||
}
|
||||
|
||||
/** Build and persist the deterministic F1 retrieval pack for a new session. */
|
||||
@@ -533,6 +536,12 @@ export class ThtRunner {
|
||||
return this.json<unknown>(["session", "show", id, "--json"], workspace);
|
||||
}
|
||||
|
||||
ensureInteractionLanguage(id: string, workspace?: string) {
|
||||
return this.json<{ interaction_language: string }>(
|
||||
["session", "ensure-interaction-language", id, "--json"], workspace,
|
||||
);
|
||||
}
|
||||
|
||||
sqlPreview(id: string, p: { limit?: number; offset?: number }, workspace?: string) {
|
||||
// No positional FILE: the harness resolves sql_final.sql from the session
|
||||
// via _session_sql_file(cfg, session_id), which respects the workspace path.
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
/** Compiled with its runtime for the host CLI: no installation, Docker or host Node required. */
|
||||
import { runWorkspaceDocuments } from "./workspaces/documents.js";
|
||||
import { runBootstrapValidation } from "./catalog/bootstrap-cli.js";
|
||||
import { runBootstrapProbes } from "./catalog/bootstrap-probes.js";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const result = args[0] === "probe" ? await runBootstrapProbes(args.slice(1))
|
||||
: args[0] === "bootstrap" ? runBootstrapValidation(args.slice(1)) : runWorkspaceDocuments(args);
|
||||
console.log(result.output);
|
||||
process.exitCode = result.status;
|
||||
@@ -18,7 +18,7 @@ export interface WorkspaceMaintenanceIo {
|
||||
writeStderr(value: string): void;
|
||||
}
|
||||
|
||||
type Command = "inspect" | "preprocess-run" | "preprocess-clear";
|
||||
type Command = "inspect" | "preprocess-run" | "preprocess-clear" | "evidence-consolidate" | "evidence-refresh" | "evidence-decide";
|
||||
|
||||
function failureResult(
|
||||
operation: string,
|
||||
@@ -65,6 +65,9 @@ function parseRequest(command: string, stdin: string): Record<string, unknown> {
|
||||
inspect: ["schemaVersion", "workspaceId"],
|
||||
"preprocess-run": ["schemaVersion", "workspaceId"],
|
||||
"preprocess-clear": ["schemaVersion", "workspaceId"],
|
||||
"evidence-consolidate": ["schemaVersion", "workspaceId"],
|
||||
"evidence-refresh": ["schemaVersion", "workspaceId"],
|
||||
"evidence-decide": ["schemaVersion", "workspaceId", "sourceId", "revision", "decision"],
|
||||
};
|
||||
const allowed = allowedByCommand[command];
|
||||
if (!allowed) throw new Error("unknown command");
|
||||
@@ -81,6 +84,16 @@ function exitCodeFor(result: WorkspaceOperationResult): number {
|
||||
|
||||
async function dispatch(command: Command, service: WorkspacePreprocessingService, request: Record<string, unknown>): Promise<WorkspaceOperationResult> {
|
||||
switch (command) {
|
||||
case "evidence-refresh":
|
||||
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "refresh" });
|
||||
case "evidence-decide":
|
||||
if (typeof request.sourceId !== "string" || !/^[a-f0-9]{64}$/.test(request.sourceId)
|
||||
|| typeof request.revision !== "string" || !/^[a-f0-9]{64}$/.test(request.revision)
|
||||
|| (request.decision !== "keep" && request.decision !== "replace")) throw new Error("invalid request");
|
||||
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "decide",
|
||||
sourceId: request.sourceId, revision: request.revision, decision: request.decision });
|
||||
case "evidence-consolidate":
|
||||
return await service.consolidateEvidence({ workspaceId: request.workspaceId as string });
|
||||
case "inspect":
|
||||
return await service.inspect({ workspaceId: request.workspaceId as string });
|
||||
case "preprocess-run":
|
||||
@@ -127,7 +140,8 @@ export async function runWorkspaceMaintenanceCli(
|
||||
|| message === "unexpected request field"
|
||||
|| message === "invalid workspace id";
|
||||
return command in {
|
||||
inspect: true, "preprocess-run": true, "preprocess-clear": true,
|
||||
inspect: true, "preprocess-run": true, "preprocess-clear": true, "evidence-consolidate": true,
|
||||
"evidence-refresh": true, "evidence-decide": true,
|
||||
} ? (requestError ? 2 : 1) : 2;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,8 +44,8 @@ const catalogSchema = z.object({
|
||||
});
|
||||
});
|
||||
|
||||
function safeCatalogError(): Error {
|
||||
return new Error("Workspace catalog is invalid");
|
||||
function safeCatalogError(cause?: unknown): Error {
|
||||
return new Error("Workspace catalog is invalid", { cause });
|
||||
}
|
||||
|
||||
export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog {
|
||||
@@ -57,7 +57,7 @@ export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog {
|
||||
return catalogSchema.parse(document.toJSON()) as WorkspaceCatalog;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === "Workspace catalog is invalid") throw error;
|
||||
throw safeCatalogError();
|
||||
throw safeCatalogError(error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
import { closeSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { parseAllDocuments, stringify } from "yaml";
|
||||
import { ZodError } from "zod";
|
||||
import { CATALOG_PATH, parseWorkspaceCatalogYaml, assertCatalogMatchesDescriptor } from "./catalog.js";
|
||||
import { parseWorkspaceYaml, type WorkspaceDescriptor } from "./schema.js";
|
||||
|
||||
interface Issue { document: string; field: string; code: string; correction: string; line?: number }
|
||||
interface Report {
|
||||
schema_version: 1;
|
||||
scope: "local-documents";
|
||||
ok: boolean;
|
||||
workspaces: { id: string; evidence: "absent" | "local-files" | "remote-deferred" }[];
|
||||
issues: Issue[];
|
||||
deferred_checks: string[];
|
||||
}
|
||||
const MAX_DOCUMENT_BYTES = 1024 * 1024;
|
||||
const usage = "tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]\ntht workspace validate --directory PATH [--json]";
|
||||
|
||||
export class DocumentError extends Error {
|
||||
constructor(readonly issue: Issue) { super(issue.correction); }
|
||||
}
|
||||
function fail(document: string, field: string, code: string, correction: string): never {
|
||||
throw new DocumentError({ document, field, code, correction });
|
||||
}
|
||||
|
||||
/** Never include parser messages or submitted values: YAML and Zod errors can contain secrets. */
|
||||
export function decode<T>(source: string, document: string, parser: (text: string) => T, contract = "workspace schema v4 or catalog schema v1"): T {
|
||||
try {
|
||||
const documents = parseAllDocuments(source, { uniqueKeys: true });
|
||||
if (documents.length !== 1) fail(document, "$", "yaml_documents", "Keep exactly one YAML document in this file.");
|
||||
const problem = [...documents[0].errors, ...documents[0].warnings][0];
|
||||
if (problem) {
|
||||
throw new DocumentError({ document, field: "$", code: "yaml_syntax", line: problem.linePos?.[0].line,
|
||||
correction: "Correct YAML syntax, remove duplicate keys and unsupported tags at the indicated line." });
|
||||
}
|
||||
return parser(source);
|
||||
} catch (error) {
|
||||
if (error instanceof DocumentError) throw error;
|
||||
const cause = error instanceof Error && error.cause instanceof ZodError ? error.cause : error;
|
||||
if (cause instanceof ZodError) {
|
||||
const issue = cause.issues[0];
|
||||
// Strict schemas produce paths containing schema-defined keys and array indices only.
|
||||
fail(document, issue.path.join(".") || "$", "schema_invalid",
|
||||
issue.code === "unrecognized_keys" ? "Remove fields not defined by the current workspace/catalog contract."
|
||||
: `Correct this field using ${contract}; check type, required value, uniqueness and allowed values.`);
|
||||
}
|
||||
fail(document, "$", "schema_invalid", "Use an authored workspace v4 descriptor; remove database configuration and keep it in the PostgreSQL Metadata Catalog.");
|
||||
}
|
||||
}
|
||||
|
||||
function stat(root: string, document: string, directory: boolean) {
|
||||
let info;
|
||||
try { info = lstatSync(join(root, document)); }
|
||||
catch { fail(document, "$", "missing_reference", "Create the referenced local file or directory and grant read access."); }
|
||||
if (info.isSymbolicLink() || (directory ? !info.isDirectory() : !info.isFile())) {
|
||||
fail(document, "$", "unsafe_reference", "Use a regular local file or directory, without symbolic links or special files.");
|
||||
}
|
||||
return info;
|
||||
}
|
||||
function readDocument(root: string, document: string): string {
|
||||
if (stat(root, document, false).size > MAX_DOCUMENT_BYTES) {
|
||||
fail(document, "$", "document_too_large", "Keep YAML documents below the 1 MiB local validation limit.");
|
||||
}
|
||||
try { return readFileSync(join(root, document), "utf8"); }
|
||||
catch { fail(document, "$", "unreadable", "Grant read access to this document and retry validation."); }
|
||||
}
|
||||
function directories(root: string, document: string) {
|
||||
stat(root, document, true);
|
||||
try { return readdirSync(join(root, document), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); }
|
||||
catch { fail(document, "$", "unreadable", "Grant read and traversal access to this directory and retry validation."); }
|
||||
}
|
||||
|
||||
function inspectEvidence(root: string, descriptor: WorkspaceDescriptor): "absent" | "local-files" | "remote-deferred" {
|
||||
const evidence = descriptor.evidence;
|
||||
if (!evidence) return "absent";
|
||||
if (evidence.source.type !== "filesystem") return "remote-deferred";
|
||||
const base = evidence.source.uri;
|
||||
stat(root, base, true);
|
||||
if (evidence.schema_version === 2) stat(root, `${base}/curated`, true);
|
||||
const patterns = evidence.source.patterns ?? ["**/*.md"];
|
||||
const literals = patterns.filter((pattern) => !/[?*\[]/.test(pattern));
|
||||
for (const pattern of literals) {
|
||||
const parts = pattern.split("/");
|
||||
for (let index = 1; index < parts.length; index++) stat(root, `${base}/${parts.slice(0, index).join("/")}`, true);
|
||||
stat(root, `${base}/${pattern}`, false);
|
||||
}
|
||||
// Inventory without following links; curated-unit interpretation remains a runtime check.
|
||||
const pending = [base];
|
||||
let count = 0;
|
||||
while (pending.length) {
|
||||
const current = pending.pop()!;
|
||||
for (const entry of directories(root, current)) {
|
||||
if (++count > 100_000) fail(base, "evidence.source", "inventory_limit", "Reduce the Evidence tree below 100,000 entries before local validation.");
|
||||
const path = `${current}/${entry.name}`;
|
||||
if (entry.isDirectory()) pending.push(path);
|
||||
else {
|
||||
const info = stat(root, path, false);
|
||||
const relative = path.slice(base.length + 1);
|
||||
// Only apply content checks to selections whose meaning is unambiguous locally.
|
||||
// Arbitrary globs are expanded by the canonical Python adapter after startup.
|
||||
const curated = evidence.schema_version === 2 && relative.startsWith("curated/") && relative.endsWith(".md");
|
||||
const selected = curated || literals.includes(relative) || (patterns.includes("**/*.md") && relative.endsWith(".md"));
|
||||
if (selected && info.size > evidence.source.max_bytes) fail(path, "evidence.source.max_bytes", "evidence_too_large", "Reduce the source file or increase the declared max_bytes limit deliberately.");
|
||||
try { closeSync(openSync(join(root, path), "r")); }
|
||||
catch { fail(path, "$", "unreadable", "Grant read access to this Evidence file and retry validation."); }
|
||||
if (curated) {
|
||||
const text = readDocument(root, path);
|
||||
const frontmatter = text.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
|
||||
if (!frontmatter) fail(path, "$", "evidence_frontmatter", "Add a YAML frontmatter block delimited by --- to the curated Markdown unit; follow the Curated Evidence contract.");
|
||||
decode(frontmatter[1], path, (source) => parseAllDocuments(source)[0].toJSON());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "local-files";
|
||||
}
|
||||
|
||||
function validate(root: string, report: Report): void {
|
||||
directories(root, ".");
|
||||
const catalog = decode(readDocument(root, CATALOG_PATH), CATALOG_PATH, parseWorkspaceCatalogYaml);
|
||||
const ids = new Set(catalog.workspaces.map((entry) => entry.id));
|
||||
for (const entry of directories(root, ".")) {
|
||||
if (entry.name === ".git") continue;
|
||||
if (entry.isSymbolicLink()) fail(".", "$", "unsafe_reference", "Replace repository-root symbolic links with regular files or directories.");
|
||||
if (entry.isDirectory() && entry.name !== "workspace-docs" && !ids.has(entry.name)) {
|
||||
fail(".", "workspaces", "unlisted_directory", "Every root directory except workspace-docs must match a catalog workspace id; remove or register the extra directory.");
|
||||
}
|
||||
if (entry.name === "workspace-docs") {
|
||||
for (const docs of directories(root, "workspace-docs")) {
|
||||
if (!ids.has(docs.name)) fail("workspace-docs", "$", "unlisted_documentation", "Keep documentation only for workspace ids listed in the catalog.");
|
||||
for (const file of directories(root, `workspace-docs/${docs.name}`)) {
|
||||
const path = `workspace-docs/${docs.name}/${file.name}`;
|
||||
if (!["README.md", "contract.env.example"].includes(file.name)) fail(`workspace-docs/${docs.name}`, "$", "unsupported_documentation", "Keep only README.md and contract.env.example in workspace-docs/<id>. Place Evidence inside the workspace directory.");
|
||||
stat(root, path, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const entry of catalog.workspaces) {
|
||||
const document = `${entry.id}/workspace.yaml`;
|
||||
try {
|
||||
stat(root, entry.id, true);
|
||||
const descriptor = decode(readDocument(root, document), document, parseWorkspaceYaml);
|
||||
try { assertCatalogMatchesDescriptor(entry, descriptor); }
|
||||
catch { fail(document, "workspace", "catalog_mismatch", "Make id, name and description identical in the catalog, descriptor and workspace directory name."); }
|
||||
const evidence = inspectEvidence(root, descriptor);
|
||||
report.workspaces.push({ id: entry.id, evidence });
|
||||
if (evidence !== "absent") report.deferred_checks.push(`${entry.id}:evidence-source-selection`, `${entry.id}:evidence-content-provenance-and-indexing`);
|
||||
if (evidence === "remote-deferred") report.deferred_checks.push(`${entry.id}:remote-evidence-access`);
|
||||
} catch (error) {
|
||||
if (error instanceof DocumentError) report.issues.push(error.issue);
|
||||
else throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function prepare(root: string, options: Map<string, string>): void {
|
||||
const id = options.get("--id");
|
||||
const name = options.get("--name");
|
||||
const language = options.get("--language") ?? "en";
|
||||
const catalog = stringify({ schema_version: 1, workspaces: [{ id, name }] });
|
||||
const descriptor = stringify({ workspace: { schema_version: 4, id, name, language } });
|
||||
decode(catalog, CATALOG_PATH, parseWorkspaceCatalogYaml);
|
||||
decode(descriptor, "workspace.yaml", parseWorkspaceYaml);
|
||||
try { mkdirSync(root); }
|
||||
catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "EEXIST") fail(".", "--directory", "destination_exists", "Choose a new directory; preparation never overwrites an existing directory or its documents.");
|
||||
fail(".", "--directory", "destination_unavailable", "Create the parent directory and grant write access, then choose a new destination.");
|
||||
}
|
||||
try {
|
||||
mkdirSync(join(root, id!));
|
||||
mkdirSync(join(root, "workspace-docs", id!), { recursive: true });
|
||||
writeFileSync(join(root, CATALOG_PATH), "# Index of workspace identities. Keep metadata identical to each descriptor.\n" + catalog, { flag: "wx" });
|
||||
writeFileSync(join(root, id!, "workspace.yaml"), "# Authored workspace v4: optional Evidence; database binding belongs to the Metadata Catalog.\n" + descriptor, { flag: "wx" });
|
||||
writeFileSync(join(root, "workspace-docs", id!, "README.md"),
|
||||
"# Workspace documents / Documenti workspace\n\n" +
|
||||
"EN: Edit thoth-workspaces.yaml and <id>/workspace.yaml together. Evidence is optional and absent by default. Add reviewed source material under <id>/evidence only when configured. Database connections and schema belong to the installation Metadata Catalog. No example databases are downloaded.\n\n" +
|
||||
"IT: Modificare insieme thoth-workspaces.yaml e <id>/workspace.yaml. Le Evidence sono facoltative e inizialmente assenti. Inserire materiale verificato in <id>/evidence solo quando configurato. Connessioni e schema dei database appartengono al Metadata Catalog dell'installazione. Nessun database di esempio viene scaricato.\n\n" +
|
||||
"Repeat / Ripetere: `tht workspace validate --directory <repository>`. Local success does not establish runtime readiness or semantic truth / Il successo locale non certifica readiness o verità semantica.\n", { flag: "wx" });
|
||||
} catch {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
fail(".", "$", "prepare_failed", "Preparation could not write the documents; check disk space and permissions, then retry with a new directory.");
|
||||
}
|
||||
}
|
||||
|
||||
export function runWorkspaceDocuments(args: string[]): { status: number; output: string } {
|
||||
const report: Report = { schema_version: 1, scope: "local-documents", ok: false, workspaces: [], issues: [], deferred_checks: ["catalog-database-binding", "database-connectivity", "runtime-preprocessing"] };
|
||||
const json = args.includes("--json");
|
||||
let status = 1;
|
||||
try {
|
||||
const [command, ...rest] = args;
|
||||
if ((command === "prepare" || command === "validate") && rest.length === 1 && rest[0] === "--help") return { status: 0, output: usage };
|
||||
const allowed = command === "prepare" ? ["--directory", "--id", "--name", "--language"] : command === "validate" ? ["--directory"] : [];
|
||||
const options = new Map<string, string>();
|
||||
const seen = new Set<string>();
|
||||
for (let index = 0; index < rest.length; index++) {
|
||||
const key = rest[index];
|
||||
if (seen.has(key)) fail("CLI", "$", "usage", usage);
|
||||
seen.add(key);
|
||||
if (key === "--json") continue;
|
||||
if (!allowed.includes(key) || !rest[index + 1] || rest[index + 1].startsWith("--")) fail("CLI", "$", "usage", usage);
|
||||
options.set(key, rest[++index]);
|
||||
}
|
||||
if (!allowed.length || !options.has("--directory") || (command === "prepare" && (!options.has("--id") || !options.has("--name")))) fail("CLI", "$", "usage", usage);
|
||||
const root = resolve(options.get("--directory")!);
|
||||
if (command === "prepare") prepare(root, options);
|
||||
validate(root, report);
|
||||
report.ok = report.issues.length === 0;
|
||||
status = report.ok ? 0 : 1;
|
||||
} catch (error) {
|
||||
report.issues.push(error instanceof DocumentError ? error.issue : { document: ".", field: "$", code: "io_error", correction: "Check local permissions and regular files, then retry; no services were started." });
|
||||
if (report.issues[0].code === "usage") status = 2;
|
||||
}
|
||||
const output = json ? JSON.stringify(report) : [
|
||||
report.ok ? "Workspace documents pass local validation." : "Workspace documents require corrections.",
|
||||
...report.issues.map((issue) => `${issue.document}${issue.line ? `:${issue.line}` : ""} [${issue.field}] ${issue.code}: ${issue.correction}`),
|
||||
...report.workspaces.map((entry) => `${entry.id}: Evidence ${entry.evidence}`),
|
||||
`Deferred until runtime: ${report.deferred_checks.join(", ")}.`,
|
||||
].join("\n");
|
||||
return { status, output };
|
||||
}
|
||||
@@ -22,6 +22,7 @@ export interface EvidencePreprocessingRequest {
|
||||
evidence: EvidenceConfig;
|
||||
job: EvidenceJobState;
|
||||
dryRun?: boolean;
|
||||
consolidate?: boolean;
|
||||
httpPrivateHostAllowlist?: readonly string[];
|
||||
}
|
||||
|
||||
@@ -56,7 +57,7 @@ function isPrivateHost(hostname: string): boolean {
|
||||
return hostname.endsWith(".internal");
|
||||
}
|
||||
|
||||
function evidencePolicy(
|
||||
export function evidencePolicy(
|
||||
evidence: EvidenceConfig,
|
||||
httpPrivateHostAllowlist?: readonly string[],
|
||||
): EvidencePreprocessingOutcome | undefined {
|
||||
@@ -95,13 +96,14 @@ function jobResult(job: EvidenceJobState): Pick<
|
||||
};
|
||||
}
|
||||
|
||||
async function runEvidenceStage(
|
||||
export async function runEvidenceStage(
|
||||
request: EvidencePreprocessingRequest,
|
||||
deps: EvidencePreprocessingDependencies,
|
||||
): Promise<EvidencePreprocessingOutcome> {
|
||||
const payload = await deps.runStage([
|
||||
"preprocess",
|
||||
"evidence",
|
||||
...(request.consolidate ? ["--consolidate"] : []),
|
||||
...(request.dryRun ? ["--dry-run"] : []),
|
||||
...(request.job.childRuns.evidence
|
||||
? ["--resume", request.job.childRuns.evidence]
|
||||
|
||||
@@ -3,6 +3,8 @@ import { renameSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
continueEvidencePreprocessing,
|
||||
evidencePolicy,
|
||||
runEvidenceStage,
|
||||
type EvidencePreprocessingDependencies,
|
||||
type EvidencePreprocessingOutcome,
|
||||
} from "./evidence/preprocessing.js";
|
||||
@@ -108,6 +110,72 @@ function baseResult(
|
||||
export class WorkspacePreprocessingService {
|
||||
constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
|
||||
|
||||
async evidenceSources(options: { workspaceId: string; action: "refresh" | "decide";
|
||||
sourceId?: string; revision?: string; decision?: "keep" | "replace"; actor?: string }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
try {
|
||||
if (options.action === "refresh") {
|
||||
const refused = evidencePolicy(runtime.workspace.evidence, this.deps.httpPrivateHostAllowlist);
|
||||
if (refused) return baseResult(runtime, "evidence refresh", "failed", refused.code);
|
||||
}
|
||||
const argv = ["evidence", "sources", options.action, "--json", "-c", "/dev/fd/3"];
|
||||
if (options.action === "decide") {
|
||||
if (!/^[a-f0-9]{64}$/.test(options.sourceId ?? "") || !/^[a-f0-9]{64}$/.test(options.revision ?? "")
|
||||
|| !["keep", "replace"].includes(options.decision ?? "")) throw new Error("Invalid source decision");
|
||||
const preflight = await this.deps.evidencePreflight(runtime.workspace);
|
||||
if (!preflight.ok) return baseResult(runtime, "evidence sources", "failed", preflight.code);
|
||||
argv.push("--source-id", options.sourceId!, "--revision", options.revision!, "--decision", options.decision!);
|
||||
}
|
||||
argv.push("--actor", options.actor ?? "installation operator");
|
||||
const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.exitCode !== 0 || payload.status !== "succeeded") throw new Error(
|
||||
typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence source operation failed");
|
||||
return baseResult(runtime, `evidence ${options.action}`, "succeeded", "ok", {
|
||||
counts: this.numberRecord(payload.counts),
|
||||
warnings: [options.action === "refresh" ? "Source comparisons are ready in Evidence management. Active content is unchanged."
|
||||
: "Source decision activated locally. Commit and push the Evidence tree manually."],
|
||||
});
|
||||
} catch (error) {
|
||||
return baseResult(runtime, `evidence ${options.action}`, "failed", "evidence_materialization_required", {
|
||||
warnings: [error instanceof Error ? error.message : "Evidence source operation failed"],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async consolidateEvidence(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
const preflight = await this.deps.evidencePreflight(runtime.workspace);
|
||||
if (!preflight.ok) return baseResult(runtime, "evidence consolidate", "failed", preflight.code);
|
||||
// Evidence has its own durable archive/corpus jobs. Do not alter Catalog readiness
|
||||
// or the full preprocessing job when publishing this one component.
|
||||
try {
|
||||
const outcome = await runEvidenceStage({ evidence: runtime.workspace.evidence,
|
||||
consolidate: true, job: { runId: randomBytes(16).toString("hex"), completedStages: [], childRuns: {} },
|
||||
}, {
|
||||
runStage: async argv => {
|
||||
const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.exitCode !== 0 || payload.status !== "succeeded") {
|
||||
return Promise.reject(new Error(typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence consolidation failed. Retry the command."));
|
||||
}
|
||||
return payload;
|
||||
},
|
||||
persistJob: () => undefined,
|
||||
evidencePreflight: async () => preflight,
|
||||
requireRunId: value => this.requireRunId(value),
|
||||
numberRecord: value => this.numberRecord(value),
|
||||
});
|
||||
return baseResult(runtime, "evidence consolidate", "succeeded", "ok", {
|
||||
...outcome, warnings: ["Evidence is active locally. Catalog and Schema readiness are unchanged. Commit and push the Evidence files manually."],
|
||||
});
|
||||
} catch (error) {
|
||||
return baseResult(runtime, "evidence consolidate", "failed", "evidence_materialization_required", {
|
||||
warnings: [error instanceof Error ? error.message : "Evidence consolidation failed. Retry the command."],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
try {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
|
||||
@@ -497,7 +497,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
rendered.workspaceRevision,
|
||||
renderedConfigObject,
|
||||
);
|
||||
const identitySuffix = inputFingerprintValue.slice(7, 23);
|
||||
// The Catalog input identity intentionally excludes representation-only changes.
|
||||
// A lease also identifies its bytes, so a new renderer never collides with an
|
||||
// immutable config produced by an earlier release for the same Catalog inputs.
|
||||
const identitySuffix = sha256(inputFingerprintValue + "\n" + publishedConfig).slice(7, 23);
|
||||
|
||||
const preprocessingRoot = ensureTrustedDirectory(join(
|
||||
options.dataRoot,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { basename, join } from "node:path";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { stringify } from "yaml";
|
||||
import { buildInstallationContract } from "./contracts.js";
|
||||
import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js";
|
||||
@@ -179,6 +179,7 @@ function renderEvidence(
|
||||
return {
|
||||
evidence: {
|
||||
...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}),
|
||||
local_archive_root: join(dirname(dirname(context.revisionContentRoot)), "repo", context.workspaceId),
|
||||
sources: [renderedSource],
|
||||
},
|
||||
vector: {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import Fastify from "fastify";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { sessionRoutes } from "../src/routes/sessions.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
test.each([
|
||||
[false, "m", 403], [false, "e", 403], [false, "reject", 204],
|
||||
[true, "m", 204], [true, "e", 204], [true, "forged", 400],
|
||||
] as const)("archive repair response enforces the responding principal (%s, %s)", async (admin, choice, status) => {
|
||||
const app = Fastify();
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "reviewer", isAdmin: admin,
|
||||
roles: admin ? ["admin"] : ["user"],
|
||||
permissions: admin ? ["session.use", "memory.manage", "evidence.manage"] : ["session.use"] };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const respond = vi.fn(() => true);
|
||||
sessionRoutes(app, {
|
||||
tht: {}, mgr: { get: () => ({ ownerKey: "test\0reviewer", bridge: {
|
||||
respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair", repair: { options: [
|
||||
{ id: "m", archive: "memory" }, { id: "e", archive: "evidence" },
|
||||
] } }),
|
||||
} }) },
|
||||
} as unknown as Parameters<typeof sessionRoutes>[1]);
|
||||
try {
|
||||
const result = await app.inject({ method: "POST", url: "/sessions/s/response",
|
||||
payload: { ui_response: { id: "gate", choices: [choice] } } });
|
||||
expect(result.statusCode).toBe(status);
|
||||
expect(respond).toHaveBeenCalledTimes(status === 204 ? 1 : 0);
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("an administrator cannot attribute a repair to another runtime's principal", async () => {
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async request => { request.principal = {
|
||||
issuer: "test", subject: "second-admin", isAdmin: true, roles: ["admin"],
|
||||
permissions: ["session.use", "memory.manage"],
|
||||
}; });
|
||||
const respond = vi.fn();
|
||||
sessionRoutes(app, { tht: {}, mgr: { get: () => ({ ownerKey: "test\0first-admin", bridge: {
|
||||
respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair",
|
||||
repair: { options: [{ id: "m", archive: "memory" }] } }),
|
||||
} }) } } as unknown as Parameters<typeof sessionRoutes>[1]);
|
||||
try {
|
||||
const result = await app.inject({ method: "POST", url: "/sessions/s/response",
|
||||
payload: { ui_response: { id: "gate", choices: ["m"] } } });
|
||||
expect(result.statusCode).toBe(409);
|
||||
expect(respond).not.toHaveBeenCalled();
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
@@ -188,6 +188,8 @@ test("roles collapse duplicates and admin contains all administrative permission
|
||||
"workspace.manage",
|
||||
"workspace.secrets.manage",
|
||||
"database.manage",
|
||||
"memory.manage",
|
||||
"evidence.manage",
|
||||
"pi.manage",
|
||||
"auth.diagnostics.read",
|
||||
]);
|
||||
|
||||
@@ -130,7 +130,7 @@ test("local login sets a non-persistent opaque session cookie and exposes only a
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/),
|
||||
|
||||
@@ -7,6 +7,8 @@ import { join } from "node:path";
|
||||
import { expandLocalHome, localPrincipal, upstreamPrincipal } from "../src/auth/principal.js";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { rolesToPermissions } from "../src/auth/config.js";
|
||||
import type { Permission, Role } from "../src/auth/types.js";
|
||||
|
||||
test("server smoke rejects retired trusted claims under OIDC authentication", () => {
|
||||
const smoke = readFileSync("../scripts/unified-deployment-smoke.sh", "utf8");
|
||||
@@ -32,7 +34,7 @@ test("local mode resolves a stable local principal", async () => {
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -74,7 +76,7 @@ test("upstream mode accepts only normalized proxy principal headers", async () =
|
||||
issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -206,15 +208,33 @@ test("the session boundary rejects tht maintenance headers outside exact loopbac
|
||||
})).statusCode).toBe(503);
|
||||
});
|
||||
|
||||
test("the session boundary touches a valid cookie session through the bounded Task 7 store operation", async () => {
|
||||
test.each<{
|
||||
name: string;
|
||||
roles: Role[];
|
||||
storedPermissions: Permission[];
|
||||
}>([
|
||||
{ name: "current user", roles: ["user"], storedPermissions: ["session.use"] },
|
||||
{
|
||||
name: "administrator signed in before archive permissions existed",
|
||||
roles: ["admin"],
|
||||
storedPermissions: rolesToPermissions(["admin"]).filter(
|
||||
(permission) => permission !== "memory.manage" && permission !== "evidence.manage",
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "user with obsolete administrative permissions",
|
||||
roles: ["user"],
|
||||
storedPermissions: ["session.use", "memory.manage", "evidence.manage"],
|
||||
},
|
||||
])("the session boundary derives current permissions and touches a valid cookie: $name", async ({ roles, storedPermissions }) => {
|
||||
const sessions = {
|
||||
resolve: vi.fn(async () => ({
|
||||
version: 1,
|
||||
issuer: "local",
|
||||
subject: "user-1",
|
||||
method: "local",
|
||||
roles: ["user"],
|
||||
permissions: ["session.use"],
|
||||
roles,
|
||||
permissions: storedPermissions,
|
||||
userAuthRevision: 1,
|
||||
authConfigRevision: "b".repeat(64),
|
||||
remembered: false,
|
||||
@@ -249,7 +269,13 @@ test("the session boundary touches a valid cookie session through the bounded Ta
|
||||
app.get("/private", async (request) => getPrincipal(request));
|
||||
|
||||
const token = "z".repeat(43);
|
||||
expect((await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } })).statusCode).toBe(200);
|
||||
const response = await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({
|
||||
roles,
|
||||
permissions: rolesToPermissions(roles),
|
||||
isAdmin: roles.includes("admin"),
|
||||
});
|
||||
expect(sessions.touch).toHaveBeenCalledWith(token);
|
||||
});
|
||||
|
||||
|
||||
@@ -124,8 +124,17 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
],
|
||||
relationships: [],
|
||||
};
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot))
|
||||
const memoryCleanupRun = await repository.createSyncRun(created.id, "columns", [], 1);
|
||||
await repository.updateSyncRun(memoryCleanupRun.id, { state: "applying", phase: "applying" });
|
||||
await expect(repository.applySchemaSync(created.id, 999, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
||||
.resolves.toBeUndefined();
|
||||
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("applying");
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
||||
.toMatchObject({ created: 4, deleted: 0 });
|
||||
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("memory_cleanup");
|
||||
await repository.interruptActiveSyncRuns();
|
||||
expect(await repository.getSyncRun(memoryCleanupRun.id))
|
||||
.toMatchObject({ state: "interrupted", phase: "memory_cleanup" });
|
||||
expect((await repository.listColumns(created.id, patients.id)).map((column) => ({
|
||||
name: column.name,
|
||||
sensitive: column.sensitive,
|
||||
@@ -233,6 +242,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
});
|
||||
expect(await repository.listSyncRuns(created.id)).toEqual([
|
||||
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
||||
expect.objectContaining({ id: memoryCleanupRun.id, phase: "memory_cleanup" }),
|
||||
]);
|
||||
|
||||
const lockedDatabase = await repository.create({
|
||||
|
||||
@@ -117,6 +117,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
});
|
||||
const introspector: CatalogSchemaIntrospector = { scan };
|
||||
const operations = new CatalogOperationCoordinator();
|
||||
const cleanup = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 0 }), stderr: "" }));
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
@@ -124,7 +125,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), {
|
||||
thtRunner: {} as never,
|
||||
thtRunner: { withPrincipal: () => ({ runWithRuntimeSnapshot: cleanup }) } as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
||||
catalogRepository: repository,
|
||||
@@ -133,7 +134,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return {
|
||||
app, repository, database: (await repository.get(created.id))!, scan, operations,
|
||||
app, repository, database: (await repository.get(created.id))!, scan, operations, cleanup,
|
||||
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
||||
};
|
||||
}
|
||||
@@ -614,6 +615,45 @@ test("waits for confirmation and rescans before applying destructive changes", a
|
||||
expect(scan).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
test("retries committed Memory cleanup with the original removals without rescanning", async () => {
|
||||
const { app, repository, database, scan, setObserved, cleanup } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const observed = snapshot();
|
||||
observed.tables = observed.tables.filter(t => t.name !== "visits");
|
||||
observed.columns = observed.columns.filter(c => c.tableName !== "visits");
|
||||
observed.relationships = [];
|
||||
setObserved(observed);
|
||||
cleanup.mockResolvedValueOnce({ code: 0, stdout: JSON.stringify({ indexed: false, deleted: 2 }), stderr: "" });
|
||||
const started = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
const waiting = await waitFor(repository, started.json().id, "awaiting_confirmation");
|
||||
expect(cleanup).not.toHaveBeenCalled();
|
||||
await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`,
|
||||
payload: { confirmationToken: waiting.confirmationToken } });
|
||||
const failed = await waitFor(repository, waiting.id, "failed");
|
||||
expect(failed).toMatchObject({ phase: "memory_cleanup", errorCode: "memory_cleanup_pending",
|
||||
plannedDiff: { deletedTables: ["visits"] } });
|
||||
expect((await repository.listTables(database.id)).map(t => t.name)).toEqual(["patients"]);
|
||||
const callsBeforeRetry = scan.mock.calls.length;
|
||||
const blocked = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
expect(blocked.statusCode).toBe(409);
|
||||
// Simulate startup recovery and an unreachable DWH after schema application.
|
||||
await repository.interruptActiveSyncRuns();
|
||||
scan.mockRejectedValue(new Error("DWH unavailable"));
|
||||
cleanup.mockResolvedValue({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 2 }), stderr: "" });
|
||||
const retried = await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/retry` });
|
||||
expect(retried.statusCode).toBe(202);
|
||||
const completed = await waitFor(repository, waiting.id, "succeeded");
|
||||
expect(completed.counts).toMatchObject({ memoryDeleted: 2 });
|
||||
expect(scan.mock.calls.length).toBe(callsBeforeRetry);
|
||||
expect(cleanup).toHaveBeenCalledTimes(2);
|
||||
expect(cleanup.mock.calls[1]).toEqual(cleanup.mock.calls[0]);
|
||||
const request = JSON.parse((cleanup.mock.calls[0] as unknown as string[])[1]!).request;
|
||||
expect(request).toMatchObject({ sync_id: waiting.id, database: "warehouse",
|
||||
schema_name: "datawarehouse", removed_tables: ["visits"] });
|
||||
});
|
||||
|
||||
test("deletes every catalog table for multiple selected databases and cascades dependent metadata", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const second = await repository.create({
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { validateDatabaseBootstrap } from "../src/catalog/bootstrap-documents.js";
|
||||
import { runBootstrapValidation } from "../src/catalog/bootstrap-cli.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
|
||||
function fixture() {
|
||||
const root = mkdtempSync(join(tmpdir(), "bootstrap-documents-")); roots.push(root);
|
||||
mkdirSync(join(root, "practice"));
|
||||
writeFileSync(join(root, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: practice, name: Practice}]\n");
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\n");
|
||||
return root;
|
||||
}
|
||||
const entry = { workspaceId: "practice", engine: "postgres", databaseName: "practice", schema: "public", binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, secretFiles: { password: "/private/operator/db-password" } };
|
||||
|
||||
test("bootstrap reuses Catalog configuration and requires one complete binding per workspace", () => {
|
||||
const root = fixture();
|
||||
const valid = validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root);
|
||||
expect(valid.secretFiles).toContainEqual({ field: "databases.0.secretFiles.password", path: "/private/operator/db-password" });
|
||||
for (const databases of [[], [entry, entry], [{ ...entry, workspaceId: "unknown" }], [{ ...entry, secretFiles: {} }], [{ ...entry, engine: "mysql" }], [{ ...entry, binding: { ...entry.binding, port: 70000 } }]]) {
|
||||
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases }, root)).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test("REST authentication, SSH credentials and optional Evidence use their declared contracts", () => {
|
||||
const root = fixture();
|
||||
const rest = { ...entry, binding: { transport: "rest_api", baseUrl: "https://data.internal", restPath: "/query", restAuth: "none" }, secretFiles: {} };
|
||||
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [rest] }, root).secretFiles).toEqual([]);
|
||||
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...rest, binding: { ...rest.binding, restAuth: "bearer" } }] }, root)).toThrow();
|
||||
const ssh = { ...entry, binding: { transport: "ssh_tunnel", username: "reader", sshHost: "bastion", sshPort: 22, sshUsername: "tunnel", sshTargetHost: "database", sshTargetPort: 5432 }, secretFiles: { password: "/password", sshPrivateKey: "/key", sshKnownHosts: "/hosts" } };
|
||||
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [ssh] }, root).warnings[0]).toContain("not NL-to-SQL");
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\nevidence:\n source:\n type: http\n uris: [https://docs.internal/manual.md]\n authentication: signed_urls_file\n");
|
||||
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root)).toThrow();
|
||||
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...entry, evidenceSecretFiles: { "evidence.signed_urls": "/urls.json" } }] }, root).secretFiles).toContainEqual({ field: "databases.0.evidenceSecretFiles.evidence.signed_urls", path: "/urls.json" });
|
||||
});
|
||||
|
||||
test("bootstrap CLI never echoes arbitrary keys from submitted secret references", () => {
|
||||
const root = fixture();
|
||||
const path = join(root, "bootstrap.yaml");
|
||||
for (const field of ["secretFiles", "evidenceSecretFiles"]) {
|
||||
writeFileSync(path, JSON.stringify({ schemaVersion: 1, databases: [{ ...entry, [field]: { PRIVATE_CREDENTIAL_SENTINEL: "/path" } }] }));
|
||||
const result = runBootstrapValidation(["--directory", root, "--bootstrap", path, "--json"]);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.output).not.toContain("PRIVATE_CREDENTIAL_SENTINEL");
|
||||
}
|
||||
});
|
||||
@@ -36,7 +36,7 @@ test("loop F1: crea sessione → SSE riceve il widget → risponde → il modell
|
||||
ollamaEnsure: async () => ({ ok: true }),
|
||||
searchPack: async () => {},
|
||||
sessionNew: async () => ({ id: "s1" }),
|
||||
sessionShow: async (_id: string) => ({ id: "s1", provider: undefined, model: undefined, thinking: undefined }),
|
||||
sessionShow: async (_id: string) => ({ id: "s1", interaction_language: "en", provider: undefined, model: undefined, thinking: undefined }),
|
||||
sessionList: async () => [],
|
||||
} as any,
|
||||
spawnFn: () => spawn("node", [FAKE, SCRIPT]) as any,
|
||||
@@ -48,7 +48,7 @@ test("loop F1: crea sessione → SSE riceve il widget → risponde → il modell
|
||||
const created = await fetch(`${base}/sessions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ workspace: "w", question: "q" }),
|
||||
body: JSON.stringify({ workspace: "w", question: "q", interactionLanguage: "en" }),
|
||||
});
|
||||
expect(created.status).toBe(200);
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import Fastify from "fastify";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { evidenceRoutes } from "../src/routes/evidence.js";
|
||||
import type { ThtRunner } from "../src/tht/tht-runner.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
const apps: ReturnType<typeof Fastify>[] = [];
|
||||
afterEach(async () => { await Promise.all(apps.splice(0).map(app => app.close())); });
|
||||
function setup(admin = true) {
|
||||
const app = Fastify(); apps.push(app);
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "curator", roles: [admin ? "admin" : "user"],
|
||||
permissions: admin ? ["evidence.manage"] : ["session.use"], isAdmin: admin };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const run = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ items: [], total: 0 }), stderr: "" }));
|
||||
const withPrincipal = vi.fn(() => ({ runWithRuntimeSnapshot: run }) as unknown as ThtRunner);
|
||||
const consolidateEvidence = vi.fn();
|
||||
const evidenceSources = vi.fn().mockResolvedValue({status: "succeeded"});
|
||||
evidenceRoutes(app, { runner: { withPrincipal }, registryRoot: "/data/registry", hostRegistryRoot: "/srv/Thoth workspaces",
|
||||
registry: { list: async () => [{ id: "sales", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/missing" }] },
|
||||
service: { consolidateEvidence, evidenceSources } });
|
||||
return { app, run, withPrincipal, principal, consolidateEvidence, evidenceSources };
|
||||
}
|
||||
test("browses complete local files with host paths, without a database runtime", async () => {
|
||||
const { app, run, withPrincipal, principal } = setup();
|
||||
const response = await app.inject("/workspaces/sales/evidence?kind=domain&concept=orders&page=2");
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(withPrincipal).toHaveBeenCalledWith(principal);
|
||||
const [argv, snapshot] = run.mock.calls[0] as unknown as [string[], string];
|
||||
expect(argv).toEqual(["evidence", "admin", "--workspace", "sales"]);
|
||||
expect(JSON.parse(snapshot)).toEqual({ root: "/data/registry/repo/sales", query: { kind: "domain", concept: "orders", page: 2 } });
|
||||
expect(response.json().location.workspace).toBe("/srv/Thoth workspaces/repo/sales");
|
||||
expect(response.json().location.git_commands[0]).toContain("git -C '/srv/Thoth workspaces/repo'");
|
||||
});
|
||||
|
||||
test("source actions require admin, bind the actor, and reject cross-workspace or arbitrary inputs", async () => {
|
||||
const denied = setup(false);
|
||||
expect((await denied.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: {action: "refresh"}})).statusCode).toBe(403);
|
||||
expect(denied.evidenceSources).not.toHaveBeenCalled();
|
||||
const allowed = setup();
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: {action: "refresh"}})).statusCode).toBe(200);
|
||||
expect(allowed.evidenceSources).toHaveBeenCalledWith({workspaceId: "sales", action: "refresh", actor: "curator"});
|
||||
const choice = {action: "decide", sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "keep"};
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: choice})).statusCode).toBe(200);
|
||||
expect(allowed.evidenceSources).toHaveBeenLastCalledWith({...choice, workspaceId: "sales", actor: "curator"});
|
||||
for (const payload of [{action: "refresh", url: "http://localhost"}, {...choice, actor: "forged"}, {...choice, revision: "../other"}]) {
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload})).statusCode).toBe(400);
|
||||
}
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/other/evidence/sources", payload: choice})).statusCode).toBe(404);
|
||||
expect(allowed.evidenceSources).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
test.each(["/", "/evidence:rule", "/consolidate"])("refuses non-admin access %s", async suffix => {
|
||||
const { app, run, consolidateEvidence } = setup(false);
|
||||
const response = await app.inject({ method: suffix === "/consolidate" ? "POST" : "GET", url: `/workspaces/sales/evidence${suffix === "/" ? "" : suffix}` });
|
||||
expect(response.statusCode).toBe(403); expect(run).not.toHaveBeenCalled(); expect(consolidateEvidence).not.toHaveBeenCalled();
|
||||
});
|
||||
test("rejects workspace escape, unknown workspace and unsupported filters", async () => {
|
||||
const { app, run } = setup();
|
||||
expect((await app.inject("/workspaces/foreign/evidence")).statusCode).toBe(404);
|
||||
expect((await app.inject("/workspaces/sales/evidence?root=/private")).statusCode).toBe(400);
|
||||
expect((await app.inject("/workspaces/sales/evidence?page_size=101")).statusCode).toBe(400);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
test("a missing unit is 404 and consolidation preserves the partial outcome", async () => {
|
||||
const { app, consolidateEvidence } = setup();
|
||||
expect((await app.inject("/workspaces/sales/evidence/evidence:missing")).statusCode).toBe(404);
|
||||
consolidateEvidence.mockResolvedValue({ status: "failed", warnings: ["Saved; retry indexing."] });
|
||||
const response = await app.inject({ method: "POST", url: "/workspaces/sales/evidence/consolidate" });
|
||||
expect(response.json()).toEqual({ status: "failed", warnings: ["Saved; retry indexing."] });
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { rootCertificates } from "node:tls";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
|
||||
const binary = process.env.THT_INSTALLATION_TEST_CLI;
|
||||
const roots: string[] = [];
|
||||
afterEach(() => roots.splice(0).forEach((path) => rmSync(path, { recursive: true, force: true })));
|
||||
function run(...args: string[]) {
|
||||
return spawnSync(binary!, [...args, "--json"], { encoding: "utf8", env: { ...process.env, PATH: "" }, input: "" });
|
||||
}
|
||||
|
||||
test.skipIf(!binary)("operator prepares, completes and repeatedly validates before any stack exists", () => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "application-documents-"))); roots.push(root);
|
||||
const workspace = join(root, "workspaces"), directory = join(root, "installation");
|
||||
expect(run("workspace", "prepare", "--directory", workspace, "--id", "practice", "--name", "Practice").status).toBe(0);
|
||||
expect(run("installation", "prepare", "--directory", directory).status).toBe(0);
|
||||
const installation = join(directory, "thothii-installation.yaml");
|
||||
const validate = () => run("--installation", installation, "installation", "validate", "--workspaces", workspace);
|
||||
expect(validate().status).toBe(1); // Visible placeholders cannot be approved.
|
||||
expect(run("installation", "credentials", "--directory", directory).status).toBe(0);
|
||||
for (const name of ["thothii-installation.yaml", "operator.env", "database-bootstrap.yaml"]) {
|
||||
const path = join(directory, name);
|
||||
writeFileSync(path, readFileSync(path, "utf8").replaceAll("CHANGE_ME", "practice"));
|
||||
}
|
||||
for (const [name, contents] of Object.entries({ "secrets.env": "OPENAI_API_KEY=PRIVATE_PROVIDER_VALUE\n", "database-password": "PRIVATE_DATABASE_VALUE", "git-credentials": "", "git-ca.pem": rootCertificates[0] })) {
|
||||
writeFileSync(join(directory, "secrets", name), contents, { mode: 0o600 });
|
||||
}
|
||||
const before = readFileSync(installation, "utf8");
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const checked = validate();
|
||||
expect(checked.stderr).toBe("");
|
||||
expect(checked.stdout).not.toContain("PRIVATE_");
|
||||
expect(checked.status, checked.stdout).toBe(0);
|
||||
expect(JSON.parse(checked.stdout).deferred_checks).toContain("release-assets");
|
||||
}
|
||||
expect(readFileSync(installation, "utf8")).toBe(before);
|
||||
writeFileSync(installation, before.replace("interaction: openai/gpt-4.1-mini", "interaction: openai/nonexistent"));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(installation, before);
|
||||
const authFile = join(directory, "secrets/pi-auth.json");
|
||||
writeFileSync(authFile, "not-json");
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(installation, before.replace("{mode: secret_env, apiKeyEnv: OPENAI_API_KEY}", "{mode: pi_auth}"));
|
||||
writeFileSync(authFile, JSON.stringify({ openai: { unrelated: true } }));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(authFile, JSON.stringify({ " OpenAI ": { type: "api_key", key: "PRIVATE_PI_KEY" } }));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(authFile, JSON.stringify({ openai: { type: "api_key", key: "PRIVATE_PI_KEY" } }));
|
||||
expect(validate().status).toBe(0);
|
||||
writeFileSync(installation, before);
|
||||
writeFileSync(authFile, "{}");
|
||||
const bootstrap = join(directory, "database-bootstrap.yaml");
|
||||
const originalBootstrap = readFileSync(bootstrap, "utf8");
|
||||
writeFileSync(join(workspace, "practice", "private-password"), "PRIVATE_DATABASE_VALUE", { mode: 0o600 });
|
||||
writeFileSync(bootstrap, originalBootstrap.replace(join(directory, "secrets/database-password"), join(workspace, "practice/private-password")));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(bootstrap, originalBootstrap);
|
||||
chmodSync(join(directory, "secrets/database-password"), 0o644);
|
||||
expect(validate().status).toBe(1);
|
||||
chmodSync(join(directory, "secrets/database-password"), 0o600);
|
||||
const env = join(directory, "operator.env");
|
||||
writeFileSync(env, readFileSync(env, "utf8") + "THOTH_HTTP_PORT=8081\n");
|
||||
expect(validate().status).toBe(1);
|
||||
}, 15_000);
|
||||
@@ -0,0 +1,43 @@
|
||||
import { createServer } from "node:http";
|
||||
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { probeBootstrapDependencies } from "../src/catalog/bootstrap-probes.js";
|
||||
import { probePublicEvidenceUrl } from "../src/catalog/evidence-probe-http.js";
|
||||
|
||||
it("refuses loopback literals and DNS answers before sending an Evidence GET", async () => {
|
||||
for (const hostname of ["[::1]", "127.0.0.1", "localhost", "[::ffff:127.0.0.1]"]) {
|
||||
await expect(probePublicEvidenceUrl(new URL(`http://${hostname}/private`))).rejects.toThrow("Evidence network policy refused");
|
||||
}
|
||||
});
|
||||
|
||||
it("authenticates a bounded read-only REST probe and blocks unavailable credentials/services", async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "tht-probe-"));
|
||||
const secret = join(directory, "key");
|
||||
writeFileSync(secret, "PRIVATE_SENTINEL", { mode: 0o600 });
|
||||
let status = 200;
|
||||
const requests: string[] = [];
|
||||
const server = createServer((req, res) => {
|
||||
requests.push(`${req.method} ${req.url}`);
|
||||
res.writeHead(req.headers.authorization === "Bearer PRIVATE_SENTINEL" ? status : 401);
|
||||
res.end("PRIVATE_SERVER_RESPONSE");
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address() as { port: number };
|
||||
const document = { schemaVersion: 1, databases: [{ workspaceId: "demo", engine: "postgres", databaseName: "demo", schema: "public", binding: { transport: "rest_api", baseUrl: `http://127.0.0.1:${address.port}`, restPath: "/health", restAuth: "bearer" }, secretFiles: { apiKey: secret } }] };
|
||||
try {
|
||||
expect((await probeBootstrapDependencies(document)).ok).toBe(true);
|
||||
status = 503;
|
||||
const failed = await probeBootstrapDependencies(document);
|
||||
expect(failed.ok).toBe(false);
|
||||
expect(JSON.stringify(failed)).not.toContain("PRIVATE");
|
||||
status = 200;
|
||||
writeFileSync(secret, "rotated-but-invalid");
|
||||
expect((await probeBootstrapDependencies(document)).ok).toBe(false);
|
||||
expect(requests).toEqual(["GET /health", "GET /health", "GET /health"]);
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
rmSync(directory, { recursive: true });
|
||||
}
|
||||
});
|
||||
@@ -56,7 +56,7 @@ test("catalog listing translates upstream Pi IDs back to canonical model keys",
|
||||
session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
};
|
||||
try {
|
||||
@@ -75,6 +75,38 @@ test("catalog listing translates upstream Pi IDs back to canonical model keys",
|
||||
}
|
||||
});
|
||||
|
||||
test("catalog listing supplies the shared DeepSeek key without requiring Pi auth", async () => {
|
||||
const script = scriptWith([{ provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro" }]);
|
||||
const secret = join(path.dirname(script), "thothii.secrets");
|
||||
writeFileSync(secret, "DEEPSEEK_API_KEY=shared-key\nOPENAI_API_KEY=unrelated-key\n", { mode: 0o600 });
|
||||
const model: RuntimeModel = {
|
||||
id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
|
||||
label: "DeepSeek V4 Pro", upstreamModel: "deepseek-v4-pro",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: false },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
};
|
||||
try {
|
||||
const lister = createPiModelLister(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
...noManagedModels, modelCatalog, loadEnabledModels: enabled(model.id),
|
||||
spawnFn: (_command, _args, options) => {
|
||||
expect(options.env.DEEPSEEK_API_KEY).toBe("shared-key");
|
||||
expect(options.env).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(options.env).not.toHaveProperty("THT_SECRETS_FILE");
|
||||
return spawn("node", [FAKE, script], { env: options.env }) as any;
|
||||
},
|
||||
});
|
||||
await expect(lister()).resolves.toEqual([{
|
||||
provider: model.provider, id: model.model, name: model.label, reasoning: false,
|
||||
}]);
|
||||
} finally {
|
||||
rmSync(path.dirname(script), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
|
||||
const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import Fastify from "fastify";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { memoryRoutes } from "../src/routes/memory.js";
|
||||
import { DEFAULT_SEMANTIC_RUNTIME } from "../src/workspaces/runtime-renderer.js";
|
||||
import type { ThtRunner } from "../src/tht/tht-runner.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
const apps: ReturnType<typeof Fastify>[] = [];
|
||||
afterEach(async () => { await Promise.all(apps.splice(0).map(app => app.close())); });
|
||||
const id = "mem-11111111-1111-4111-8111-111111111111";
|
||||
const input = { family: "domain_clarification", subject: "Order", scope: "Sales" };
|
||||
|
||||
function setup(admin = true) {
|
||||
const app = Fastify(); apps.push(app);
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "operator", roles: [admin ? "admin" : "user"],
|
||||
permissions: admin ? ["memory.manage"] : ["session.use"], isAdmin: admin };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const run = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ items: [], total: 0 }), stderr: "" }));
|
||||
const bound = { runWithRuntimeSnapshot: run } as unknown as ThtRunner;
|
||||
const withPrincipal = vi.fn(() => bound);
|
||||
memoryRoutes(app, { runner: { withPrincipal }, runtime: DEFAULT_SEMANTIC_RUNTIME,
|
||||
registry: {
|
||||
list: async () => [{ id: "sales", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/missing" }],
|
||||
read: vi.fn().mockResolvedValue({ workspace: { workspace: { language: "it" } } }),
|
||||
} });
|
||||
return { app, run, withPrincipal, principal };
|
||||
}
|
||||
|
||||
test("list binds principal and workspace without requiring a DWH runtime", async () => {
|
||||
const { app, run, withPrincipal, principal } = setup();
|
||||
const response = await app.inject("/workspaces/sales/memory?q=Order&page=2&column=id");
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(withPrincipal).toHaveBeenCalledWith(principal);
|
||||
const [args, snapshot] = run.mock.calls[0] as unknown as [string[], string];
|
||||
expect(args).toEqual(["memory", "admin", "--workspace", "sales"]);
|
||||
expect(JSON.parse(snapshot)).toMatchObject({ action: "list", request: { q: "Order", page: 2, column: "id" } });
|
||||
expect(JSON.parse(snapshot).runtime.memoryLanguage).toBe("it");
|
||||
});
|
||||
|
||||
test.each([
|
||||
["GET", ""], ["POST", ""], ["GET", "/pending"], ["GET", `/${id}`],
|
||||
["PUT", `/${id}`], ["DELETE", `/${id}`], ["POST", `/${id}/retry`],
|
||||
] as const)("non-admin cannot %s %s", async (method, suffix) => {
|
||||
const { app, run } = setup(false);
|
||||
const response = await app.inject({ method, url: `/workspaces/sales/memory${suffix}`,
|
||||
...(method === "PUT" || method === "POST" && suffix === "" ? { payload: input } : {}) });
|
||||
expect(response.statusCode).toBe(403); expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("unknown workspace and invalid input do not invoke the harness", async () => {
|
||||
const { app, run } = setup();
|
||||
expect((await app.inject("/workspaces/missing/memory")).statusCode).toBe(404);
|
||||
expect((await app.inject("/workspaces/sales/memory?page_size=101")).statusCode).toBe(400);
|
||||
expect((await app.inject({ method: "POST", url: "/workspaces/sales/memory",
|
||||
payload: { ...input, workspace_id: "foreign" } })).statusCode).toBe(400);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("create preserves saved-but-unindexed outcome, update carries complete related changes", async () => {
|
||||
const { app, run } = setup();
|
||||
run.mockResolvedValue({ code: 0, stdout: JSON.stringify({ id, saved: true, indexed: false }), stderr: "" });
|
||||
const result = await app.inject({ method: "POST", url: "/workspaces/sales/memory", payload: input });
|
||||
expect(result.statusCode).toBe(201); expect(result.json()).toEqual({ id, saved: true, indexed: false });
|
||||
await app.inject({ method: "PUT", url: `/workspaces/sales/memory/${id}`, payload: {
|
||||
...input, links: [{ target_id: id, meaning: "Related" }], dependencies: [{ database: "dwh" }],
|
||||
} });
|
||||
const [, snapshot] = run.mock.calls[1] as unknown as [string[], string];
|
||||
expect(JSON.parse(snapshot)).toMatchObject({ action: "update", request: { id, card: {
|
||||
links: [{ target_id: id, meaning: "Related" }], dependencies: [{ database: "dwh" }],
|
||||
} } });
|
||||
});
|
||||
|
||||
test.each([["memory_not_found", 404], ["memory_conflict", 409], ["memory_unavailable", 503]])(
|
||||
"maps %s without leaking stderr", async (code, status) => {
|
||||
const { app, run } = setup();
|
||||
run.mockResolvedValue({ code: 1, stdout: JSON.stringify({ code, message: "sensitive detail" }), stderr: "secret" });
|
||||
const result = await app.inject("/workspaces/sales/memory");
|
||||
expect(result.statusCode).toBe(status); expect(result.json().code).toBe(code);
|
||||
expect(result.body).not.toMatch(/secret|sensitive/);
|
||||
},
|
||||
);
|
||||
@@ -20,9 +20,8 @@ function runtimeCatalog(overrides: Record<string, unknown> = {}, secrets = "OPEN
|
||||
const catalogFile = join(root, "catalog.json");
|
||||
const secretsFile = join(root, "thothii.secrets");
|
||||
const catalog = {
|
||||
schemaVersion: 1,
|
||||
defaultSession: "zai/glm-5.3",
|
||||
defaultMetadataGeneration: "zai/glm-5.3",
|
||||
schemaVersion: 2,
|
||||
defaultInteraction: "zai/glm-5.3",
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
models: [
|
||||
{
|
||||
@@ -55,8 +54,8 @@ test("loads session default and safe metadata choices from the normalized runtim
|
||||
const runtime = loadRuntimeModelCatalog(catalogFile);
|
||||
const metadata = loadMetadataGenerationModels({ catalogFile, secretsFile });
|
||||
|
||||
expect(runtime.defaultSession).toBe("zai/glm-5.3");
|
||||
expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(true);
|
||||
expect(runtime.defaultInteraction).toBe("zai/glm-5.3");
|
||||
expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(false);
|
||||
expect(metadata.catalog()).toEqual({
|
||||
models: [{ id: "zai/glm-5.3", label: "GLM 5.3" }],
|
||||
default: "zai/glm-5.3",
|
||||
@@ -69,14 +68,65 @@ test("loads session default and safe metadata choices from the normalized runtim
|
||||
expect(() => metadata.resolve("zai/missing")).toThrow(MetadataGenerationModelUnavailableError);
|
||||
});
|
||||
|
||||
test("DeepSeek uses one identity and bundle credential for native Pi and LiteLLM", () => {
|
||||
const models = ["deepseek-v4-pro", "deepseek-v4-flash"].map((model) => ({
|
||||
id: `deepseek/${model}`, provider: "deepseek", model, label: model, upstreamModel: model,
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" }, metadataAdapter: { litellmProvider: "deepseek" },
|
||||
session: { reasoning: false }, metadataGeneration: { disableThinking: false },
|
||||
}));
|
||||
const files = runtimeCatalog({ defaultInteraction: models[0].id, models }, "DEEPSEEK_API_KEY=shared-key\n");
|
||||
const runtime = loadRuntimeModelCatalog(files.catalogFile);
|
||||
const metadata = loadMetadataGenerationModels(files);
|
||||
expect(runtime.sessionModels().map((model) => model.id)).toEqual(metadata.catalog().models.map((model) => model.id));
|
||||
expect(metadata.catalog().default).toBe(runtime.defaultInteraction);
|
||||
for (const model of models) {
|
||||
expect(metadata.resolve(model.id)).toMatchObject({
|
||||
id: model.id, provider: "deepseek", model: model.model,
|
||||
apiKeyEnv: "DEEPSEEK_API_KEY", apiKey: "shared-key",
|
||||
});
|
||||
}
|
||||
expect(() => metadata.resolve("deepseek-metadata/deepseek-v4-pro"))
|
||||
.toThrow(MetadataGenerationModelUnavailableError);
|
||||
});
|
||||
|
||||
test("returns empty catalogs when no runtime projection is configured", () => {
|
||||
expect(loadRuntimeModelCatalog().defaultSession).toBeNull();
|
||||
expect(loadRuntimeModelCatalog().defaultInteraction).toBeNull();
|
||||
expect(loadMetadataGenerationModels({}).catalog()).toEqual({ models: [], default: null });
|
||||
});
|
||||
|
||||
test.each([
|
||||
["qwen-chat-template", true, true],
|
||||
["qwen", true, true],
|
||||
["qwen-typo", true, false],
|
||||
["qwen-chat-template", false, false],
|
||||
])("validates Qwen thinking format %s with reasoning=%s", (thinkingFormat, reasoning, valid) => {
|
||||
const { catalogFile } = runtimeCatalog({
|
||||
defaultInteraction: "local/qwen",
|
||||
models: [{
|
||||
id: "local/qwen", provider: "local", model: "qwen", label: "Qwen",
|
||||
upstreamModel: "qwen", endpoint: { baseUrl: "http://localhost:8000/v1" },
|
||||
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
|
||||
session: {
|
||||
reasoning, contextWindow: 32768, maxTokens: 8192,
|
||||
compatibility: {
|
||||
supportsDeveloperRole: false, supportsReasoningEffort: false,
|
||||
supportsStore: false, maxTokensField: "max_tokens", thinkingFormat,
|
||||
},
|
||||
},
|
||||
}],
|
||||
});
|
||||
if (valid) {
|
||||
expect(loadRuntimeModelCatalog(catalogFile).sessionModels()[0].session?.compatibility)
|
||||
.toMatchObject({ thinkingFormat });
|
||||
} else {
|
||||
expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid");
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects a drifted default and an unprotected projection", () => {
|
||||
const drifted = runtimeCatalog({ defaultSession: "zai/missing" });
|
||||
expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("session default is invalid");
|
||||
const drifted = runtimeCatalog({ defaultInteraction: "zai/missing" });
|
||||
expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("interaction default is invalid");
|
||||
|
||||
const unprotected = runtimeCatalog();
|
||||
chmodSync(unprotected.catalogFile, 0o666);
|
||||
@@ -85,7 +135,7 @@ test("rejects a drifted default and an unprotected projection", () => {
|
||||
|
||||
test("rejects authentication semantics that cannot come from the installation catalog", () => {
|
||||
const invalid = runtimeCatalog({
|
||||
defaultMetadataGeneration: undefined,
|
||||
defaultInteraction: undefined,
|
||||
models: [{
|
||||
id: "zai/glm-5.3",
|
||||
provider: "zai",
|
||||
@@ -112,3 +162,13 @@ test("splits canonical session identities without provider aliases", () => {
|
||||
expect(splitCanonicalModelId("zai/glm-5.3")).toEqual({ provider: "zai", model: "glm-5.3" });
|
||||
expect(() => splitCanonicalModelId("glm-5.3")).toThrow("model identity is invalid");
|
||||
});
|
||||
|
||||
test("rejects a default supported by only one configured use", () => {
|
||||
const { catalogFile } = runtimeCatalog({ defaultInteraction: "deepseek/deepseek-v4-pro" });
|
||||
expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("interaction default is invalid");
|
||||
});
|
||||
|
||||
test("requires regenerated runtime schema v2 rather than interpreting two legacy defaults", () => {
|
||||
const { catalogFile } = runtimeCatalog({ schemaVersion: 1, defaultSession: "zai/glm-5.3" });
|
||||
expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid");
|
||||
});
|
||||
|
||||
@@ -36,6 +36,10 @@ vi.mock("../src/tht/tht-runner.js", () => ({
|
||||
|
||||
vi.mock("../src/workspaces/registry.js", () => ({
|
||||
WorkspaceRegistry: class {
|
||||
async pull() {
|
||||
return { branch: "main", head: "a".repeat(40), ahead: 0, behind: 0, degraded: false };
|
||||
}
|
||||
|
||||
async listRetainedSnapshots() {
|
||||
return [{ snapshotPath: "/data/workspace-registry/snapshots/revision/workspace.yaml" }];
|
||||
}
|
||||
@@ -98,3 +102,13 @@ test("workflow doctor gives schema-v4 runtime rendering a live Catalog repositor
|
||||
expect(fakes.releaseWorkspaceRuntime).toHaveBeenCalledOnce();
|
||||
expect(fakes.catalogRepository.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("workspace pull exposes only safe Git status", async () => {
|
||||
await expect(runOperatorAction("workspace-pull", config)).resolves.toEqual({
|
||||
ready: true,
|
||||
status: "succeeded",
|
||||
branch: "main",
|
||||
head: "a".repeat(40),
|
||||
degraded: false,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, test, vi } from "vitest";
|
||||
@@ -7,7 +7,34 @@ import {
|
||||
createPiManagement,
|
||||
type PiExecFile,
|
||||
} from "../src/pi/management.js";
|
||||
import type { RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
|
||||
import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
|
||||
|
||||
test.each([false, true])("catalog credential status ignores legacy Pi auth, missing bundle key: %s", async (missingKey) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "tht-catalog-status-"));
|
||||
writeFileSync(join(root, "auth.json"), JSON.stringify({ deepseek: { type: "api_key", key: "stale-key" } }), { mode: 0o600 });
|
||||
const secret = join(root, "thothii.secrets");
|
||||
writeFileSync(secret, missingKey ? "THT_MODEL_API_KEY=legacy-key\n" : "DEEPSEEK_API_KEY=shared-key\n", { mode: 0o600 });
|
||||
vi.stubEnv("PI_CODING_AGENT_DIR", root);
|
||||
const model: RuntimeModel = {
|
||||
id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
|
||||
label: "DeepSeek", upstreamModel: "deepseek-v4-pro",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: false },
|
||||
};
|
||||
try {
|
||||
const service = createPiManagement(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
modelCatalog: {
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
},
|
||||
execute: successfulExec([]), readSettings: () => ({ thinking: "medium" }),
|
||||
});
|
||||
expect((await service.status()).credentials).toBe(missingKey ? "missing" : "present");
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-management-")), "settings.json")) {
|
||||
return loadConfig({
|
||||
@@ -15,12 +42,12 @@ function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-manage
|
||||
SETTINGS_FILE: settingsFile,
|
||||
PI_BIN: "/usr/local/bin/pi",
|
||||
PI_MANAGEMENT_TIMEOUT_MS: "750",
|
||||
THT_HOST_PLATFORM: "linux",
|
||||
});
|
||||
}
|
||||
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: "zai/glm-5.2",
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: "zai/glm-5.2",
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
sessionModels: () => [],
|
||||
metadataModels: () => [],
|
||||
@@ -34,6 +61,16 @@ function successfulExec(calls: Array<{ command: string; args: string[]; timeout:
|
||||
};
|
||||
}
|
||||
|
||||
test.each([
|
||||
["linux", "linux"], ["darwin", "macos"], ["windows", "windows"],
|
||||
])("reports installation host %s independently of the backend container OS", async (host, expected) => {
|
||||
const service = createPiManagement(loadConfig({ THT_HOST_PLATFORM: host }), {
|
||||
modelCatalog, execute: successfulExec([]), readSettings: () => ({ thinking: "medium" }),
|
||||
credentialStatus: () => "missing",
|
||||
});
|
||||
expect((await service.status()).hostPlatform).toBe(expected);
|
||||
});
|
||||
|
||||
// Catches a Pi executable that emits unexpected text or is invoked through a shell, which could
|
||||
// turn a version display into a command-injection or information-disclosure surface.
|
||||
test("status parses only a Pi version from a fixed execFile argument array", async () => {
|
||||
@@ -47,6 +84,7 @@ test("status parses only a Pi version from a fixed execFile argument array", asy
|
||||
});
|
||||
|
||||
await expect(service.status()).resolves.toEqual({
|
||||
hostPlatform: "linux",
|
||||
version: "0.80.3",
|
||||
ready: true,
|
||||
credentials: "missing",
|
||||
@@ -78,6 +116,7 @@ test.each(["present", "missing"] as const)(
|
||||
|
||||
const status = await service.status();
|
||||
expect(status).toEqual({
|
||||
hostPlatform: "linux",
|
||||
version: "0.80.3",
|
||||
ready: true,
|
||||
credentials,
|
||||
|
||||
@@ -190,6 +190,29 @@ test("Pi receives the leased workspace runtime config and releases it on direct
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("Pi language launch hint comes from the session options and never ambient environment", () => {
|
||||
const previous = process.env.THT_INTERACTION_LANGUAGE;
|
||||
process.env.THT_INTERACTION_LANGUAGE = "it";
|
||||
const environments: NodeJS.ProcessEnv[] = [];
|
||||
const mgr = new PiProcessManager(loadConfig({}), {
|
||||
spawnFn: (_command, _args, options) => {
|
||||
environments.push(options.env);
|
||||
return recordingChild() as any;
|
||||
},
|
||||
});
|
||||
try {
|
||||
mgr.createFor("explicit-language", { interactionLanguage: "en" });
|
||||
mgr.teardown("explicit-language");
|
||||
mgr.createFor("manifest-resolved-in-gate");
|
||||
mgr.teardown("manifest-resolved-in-gate");
|
||||
expect(environments[0].THT_INTERACTION_LANGUAGE).toBe("en");
|
||||
expect(environments[1]).not.toHaveProperty("THT_INTERACTION_LANGUAGE");
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.THT_INTERACTION_LANGUAGE;
|
||||
else process.env.THT_INTERACTION_LANGUAGE = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test("a close-only child event releases its temporary Pi agent snapshot", () => {
|
||||
const child = recordingChild();
|
||||
let snapshotDir: string | undefined;
|
||||
@@ -642,27 +665,33 @@ test("session Pi spawn reads the single secret bundle and scrubs its path", asyn
|
||||
}
|
||||
});
|
||||
|
||||
test("session Pi spawn resolves the selected catalog credential from the secret bundle", () => {
|
||||
test.each([false, true])("session Pi uses the catalog bundle despite stale Pi auth: %s", (missingKey) => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "thothii-catalog-credential-"));
|
||||
const agentDir = path.join(root, "agent");
|
||||
mkdirSync(agentDir, { mode: 0o700 });
|
||||
writeFileSync(path.join(agentDir, "auth.json"), "{}\n", { mode: 0o600 });
|
||||
const originalAuth = JSON.stringify({
|
||||
deepseek: { type: "api_key", key: "stale-key" },
|
||||
anthropic: { type: "api_key", key: "unrelated-key" },
|
||||
});
|
||||
writeFileSync(path.join(agentDir, "auth.json"), originalAuth, { mode: 0o600 });
|
||||
writeFileSync(path.join(agentDir, "models.json"), '{"providers":{}}\n', { mode: 0o600 });
|
||||
const secret = path.join(root, "thothii.secrets");
|
||||
writeFileSync(secret, "ZAI_API_KEY=catalog-secret\nTHT_MODEL_API_KEY=legacy-secret\n", { mode: 0o600 });
|
||||
writeFileSync(secret, missingKey ? "THT_MODEL_API_KEY=legacy-secret\n"
|
||||
: "DEEPSEEK_API_KEY=catalog-secret\nTHT_MODEL_API_KEY=legacy-secret\n", { mode: 0o600 });
|
||||
const legacyKey = path.join(root, "legacy-key");
|
||||
writeFileSync(legacyKey, "legacy-file-key", { mode: 0o600 });
|
||||
const model: RuntimeModel = {
|
||||
id: "openai/test-model",
|
||||
provider: "openai",
|
||||
model: "test-model",
|
||||
label: "Test model",
|
||||
upstreamModel: "test-model",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "ZAI_API_KEY" },
|
||||
id: "deepseek/deepseek-v4-pro",
|
||||
provider: "deepseek",
|
||||
model: "deepseek-v4-pro",
|
||||
label: "DeepSeek V4 Pro",
|
||||
upstreamModel: "deepseek-v4-pro",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" },
|
||||
session: { reasoning: false },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: model.id,
|
||||
embedding: null,
|
||||
sessionModels: () => [model],
|
||||
metadataModels: () => [],
|
||||
@@ -672,17 +701,26 @@ test("session Pi spawn resolves the selected catalog credential from the secret
|
||||
const child = recordingChild();
|
||||
child.stderr.resume = () => {};
|
||||
vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
|
||||
const mgr = new PiProcessManager(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
const mgr = new PiProcessManager(loadConfig({ THT_SECRETS_FILE: secret, THT_MODEL_API_KEY_FILE: legacyKey }), {
|
||||
modelCatalog,
|
||||
authProviders: () => new Set(),
|
||||
authProviders: () => new Set(["deepseek"]),
|
||||
spawnFn: (...args: any[]) => { calls.push(args); return child as any; },
|
||||
});
|
||||
try {
|
||||
mgr.createFor("catalog-credential", { provider: "openai", model: "test-model" });
|
||||
expect(calls[0][2].env.ZAI_API_KEY).toBe("catalog-secret");
|
||||
const create = () => mgr.createFor("catalog-credential", { provider: model.provider, model: model.model });
|
||||
if (missingKey) {
|
||||
expect(create).toThrow("model provider credential is unavailable");
|
||||
expect(calls).toHaveLength(0);
|
||||
return;
|
||||
}
|
||||
create();
|
||||
expect(calls[0][2].env.DEEPSEEK_API_KEY).toBe("catalog-secret");
|
||||
expect(calls[0][2].env).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(calls[0][2].env).not.toHaveProperty("THT_MODEL_API_KEY");
|
||||
expect(JSON.parse(readFileSync(path.join(calls[0][2].env.PI_CODING_AGENT_DIR, "auth.json"), "utf8")))
|
||||
.toEqual({ anthropic: { type: "api_key", key: "unrelated-key" } });
|
||||
} finally {
|
||||
expect(readFileSync(path.join(agentDir, "auth.json"), "utf8")).toBe(originalAuth);
|
||||
mgr.teardown("catalog-credential");
|
||||
vi.unstubAllEnvs();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
@@ -750,7 +788,7 @@ test("set_model translates a canonical catalog key to its upstream Pi model ID",
|
||||
session: { reasoning: false, contextWindow: 32768, maxTokens: 8192 },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
};
|
||||
const mgr = new PiProcessManager(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
|
||||
|
||||
@@ -61,20 +61,22 @@ test("provider smoke resolves the selected catalog credential from the secret bu
|
||||
session: { reasoning: false },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: model.id,
|
||||
embedding: null,
|
||||
sessionModels: () => [model],
|
||||
metadataModels: () => [],
|
||||
hasSession: (id) => id === model.id,
|
||||
};
|
||||
let spawnEnv: NodeJS.ProcessEnv | undefined;
|
||||
const readAuthStore = vi.fn(() => JSON.stringify({ openai: { type: "api_key", key: "stale-key" } }));
|
||||
const smoke = createPiProviderSmoke(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
modelCatalog,
|
||||
authProviders: () => new Set(),
|
||||
authProviders: () => new Set(["openai"]),
|
||||
readAuthStore,
|
||||
readModelsStore: () => undefined,
|
||||
spawnFn: (_command, _args, options) => {
|
||||
spawnEnv = options.env;
|
||||
expect(existsSync(join(options.env.PI_CODING_AGENT_DIR!, "auth.json"))).toBe(false);
|
||||
return successfulProviderChild();
|
||||
},
|
||||
});
|
||||
@@ -85,6 +87,7 @@ test("provider smoke resolves the selected catalog credential from the secret bu
|
||||
expect(spawnEnv?.ZAI_API_KEY).toBe("catalog-secret");
|
||||
expect(spawnEnv).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(spawnEnv).not.toHaveProperty("THT_MODEL_API_KEY");
|
||||
expect(readAuthStore).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -306,7 +309,7 @@ test("provider smoke makes one configured request from an isolated no-capability
|
||||
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true },
|
||||
};
|
||||
const smokeCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: smokeModel.id, defaultMetadataGeneration: null, embedding: null,
|
||||
defaultInteraction: smokeModel.id, embedding: null,
|
||||
sessionModels: () => [smokeModel], metadataModels: () => [],
|
||||
hasSession: (id) => id === smokeModel.id,
|
||||
};
|
||||
|
||||
@@ -27,10 +27,9 @@ function operationalWorkspace(id = "default") {
|
||||
} as const;
|
||||
}
|
||||
|
||||
function sessionCatalog(defaultSession = "zai/glm-5.2", available = [defaultSession]) {
|
||||
function sessionCatalog(defaultInteraction = "zai/glm-5.2", available = [defaultInteraction]) {
|
||||
return {
|
||||
defaultSession,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction,
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
sessionModels: () => [],
|
||||
metadataModels: () => [],
|
||||
@@ -54,7 +53,11 @@ const defaultWorkspaceRegistry = {
|
||||
|
||||
function buildApp(config: Parameters<typeof buildRealApp>[0], deps: Record<string, unknown> = {}) {
|
||||
const thtRunner = deps.thtRunner
|
||||
? { qdrantEnsure: async () => ({ ok: true }), ...(deps.thtRunner as object) }
|
||||
? {
|
||||
qdrantEnsure: async () => ({ ok: true }),
|
||||
ensureInteractionLanguage: async () => ({ interaction_language: "en" }),
|
||||
...(deps.thtRunner as object),
|
||||
}
|
||||
: undefined;
|
||||
return buildRealApp(config, {
|
||||
workspaceRuntimeSupport: () => true,
|
||||
@@ -89,6 +92,156 @@ const aliceHeaders = {
|
||||
"x-thoth-is-admin": "0",
|
||||
};
|
||||
|
||||
test.each([undefined, null, "", "en--US", "en_US", "en\nIGNORE", "en<script>", 42])(
|
||||
"new sessions reject invalid interaction language %s before persistence", async (interactionLanguage) => {
|
||||
const app = mutApp({ sessionNew: async () => { throw new Error("must not create"); } });
|
||||
try {
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "Pazienti", interactionLanguage },
|
||||
});
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(response.json()).toMatchObject({ code: "invalid_interaction_language" });
|
||||
} finally { await app.close(); }
|
||||
},
|
||||
);
|
||||
|
||||
test.each([["en", "en"], ["fr-FR", "fr-FR"], ["FR-fr", "fr-FR"]])(
|
||||
"new session forwards canonical interaction language %s without changing the question", async (language, canonical) => {
|
||||
let created: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionNew: async (options: any) => { created = options; return { id: "language" }; },
|
||||
searchPack: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined, createFor: () => ({ bridge: { onClientEvent: () => {} } }),
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "default" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "Pazienti", interactionLanguage: language },
|
||||
});
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(created).toMatchObject({ question: "Pazienti", interactionLanguage: canonical });
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume rejects browser interaction language overrides", async () => {
|
||||
const app = mutApp({ sessionShow: async () => ({ status: "open", interaction_language: "it" }) });
|
||||
try {
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions/s1/resume", payload: { interactionLanguage: "en" },
|
||||
});
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(response.json()).toMatchObject({ code: "interaction_language_pinned" });
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("new session starts Pi with the question language persisted by the harness", async () => {
|
||||
let runtime: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionNew: async () => ({ id: "english-question", interaction_language: "en" }),
|
||||
searchPack: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: (_id: string, options: any) => {
|
||||
runtime = options;
|
||||
return { bridge: { onClientEvent: () => {} } };
|
||||
},
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "default" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: {
|
||||
question: "How many patients were admitted last year?", interactionLanguage: "it",
|
||||
} });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(runtime.interactionLanguage).toBe("en");
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume pins legacy interaction language using the resolved harness workspace", async () => {
|
||||
let pinnedWorkspace: string | undefined;
|
||||
let runtime: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ id: "legacy", status: "closed" }),
|
||||
ensureInteractionLanguage: async (_id: string, workspace: string) => {
|
||||
pinnedWorkspace = workspace;
|
||||
return { interaction_language: "it" };
|
||||
},
|
||||
reopenSession: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: (_id: string, options: any) => {
|
||||
runtime = options;
|
||||
return { bridge: { onClientEvent: () => {} } };
|
||||
},
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "other-browser-workspace" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/legacy/resume" });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(pinnedWorkspace).toContain("/default.yaml");
|
||||
expect(runtime).toMatchObject({ interactionLanguage: "it", mode: "resume" });
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume retains persisted interaction language despite different browser settings", async () => {
|
||||
let options: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ id: "saved", status: "closed", interaction_language: "it" }),
|
||||
ensureInteractionLanguage: async () => { throw new Error("language is already pinned"); },
|
||||
reopenSession: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: (_id: string, value: any) => {
|
||||
options = value;
|
||||
return { bridge: { onClientEvent: () => {} } };
|
||||
},
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "english-workspace", locale: "en" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/saved/resume" });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(options.interactionLanguage).toBe("it");
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume cannot start Pi if legacy interaction language cannot be persisted", async () => {
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ id: "legacy", status: "closed" }),
|
||||
ensureInteractionLanguage: async () => { throw new Error("private storage details"); },
|
||||
reopenSession: async () => { throw new Error("must not reopen"); },
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: { createFor: () => { throw new Error("must not spawn"); } },
|
||||
getSettings: () => ({ workspace: "default" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/legacy/resume" });
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.body).not.toContain("private storage details");
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("upstream requests without a principal fail before a Pi runtime can be created", async () => {
|
||||
let created = false;
|
||||
const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), {
|
||||
@@ -96,7 +249,7 @@ test("upstream requests without a principal fail before a Pi runtime can be crea
|
||||
thtRunner: {} as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(401);
|
||||
expect(created).toBe(false);
|
||||
@@ -134,7 +287,7 @@ test("maintenance rejects new and resumed session admission without interrupting
|
||||
});
|
||||
|
||||
const create = await app.inject({
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { question: "q" },
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
const resume = await app.inject({ method: "POST", url: "/sessions/open/resume", headers: aliceHeaders });
|
||||
|
||||
@@ -155,7 +308,7 @@ test("a durable maintenance marker initializes admission closed after backend re
|
||||
AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness", THT_MAINTENANCE_FILE: marker,
|
||||
}), { thtRunner: {} as any });
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { question: "q" },
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.json()).toMatchObject({ code: "maintenance" });
|
||||
@@ -493,7 +646,7 @@ test("new sessions are created through the authenticated principal, not a client
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders,
|
||||
payload: { question: "q", owner: "mallory" },
|
||||
payload: { interactionLanguage: "en", question: "q", owner: "mallory" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
@@ -510,7 +663,7 @@ test("new sessions reject the client legacy workspace field unless local legacy
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "q", workspace: "legacy" },
|
||||
method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q", workspace: "legacy" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
@@ -530,7 +683,7 @@ test("explicit local legacy mode permits the unpinned client workspace request",
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "q", workspace: "legacy" },
|
||||
method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q", workspace: "legacy" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
@@ -568,7 +721,7 @@ test("creates a session from the active immutable workspace revision", async ()
|
||||
|
||||
await app.inject({
|
||||
method: "POST", url: "/sessions",
|
||||
payload: { question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" },
|
||||
payload: { interactionLanguage: "en", question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" },
|
||||
});
|
||||
|
||||
expect(sessionNew).toHaveBeenCalledWith(expect.objectContaining({
|
||||
@@ -612,7 +765,7 @@ test("hands one Catalog-backed runtime to both retrieval and Pi", async () => {
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "Which users placed orders?", workspaceId: "default" },
|
||||
payload: { interactionLanguage: "en", question: "Which users placed orders?", workspaceId: "default" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
@@ -688,7 +841,7 @@ test("refuses core admission when the workspace preprocessing fingerprint is sta
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "q", workspaceId: "default" },
|
||||
payload: { interactionLanguage: "en", question: "q", workspaceId: "default" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
@@ -740,7 +893,7 @@ test("rejects an SSH-only Catalog binding before persisting or starting a sessio
|
||||
} as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toMatchObject({ code: "workspace_not_activatable" });
|
||||
@@ -780,7 +933,7 @@ test("hands a revision lease to retention only after the session manifest is dur
|
||||
workspaceRegistry: { acquireSessionRevision } as any,
|
||||
});
|
||||
|
||||
const request = app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const request = app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(markPersisted).not.toHaveBeenCalled();
|
||||
expect(abort).not.toHaveBeenCalled();
|
||||
@@ -811,7 +964,7 @@ test("creates a session from the configured default workspace revision when work
|
||||
workspaceRegistry: registry as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(registry.read).toHaveBeenCalledWith("psd-clinical");
|
||||
expect(sessionNew).toHaveBeenCalledWith(expect.objectContaining({
|
||||
@@ -897,7 +1050,7 @@ test("session lifecycle locates a B session when installation default is A", asy
|
||||
} as any,
|
||||
});
|
||||
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: {
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en",
|
||||
question: "B question", workspaceId: "b-workspace", provider: "zai", model: "glm-5.2", thinking: "low",
|
||||
} })).statusCode).toBe(200);
|
||||
expect((await app.inject({ method: "GET", url: "/sessions" })).json()).toEqual([
|
||||
@@ -909,7 +1062,7 @@ test("session lifecycle locates a B session when installation default is A", asy
|
||||
|
||||
active = undefined;
|
||||
expect((await app.inject({ method: "POST", url: "/sessions/session-b/resume" })).json())
|
||||
.toEqual({ id: "session-b", alreadyActive: false });
|
||||
.toEqual({ id: "session-b", alreadyActive: false, workspaceId: "b-workspace" });
|
||||
expect(calls).toContain(`new:${bPath}`);
|
||||
expect(calls).toContain(`list:${bPath}`);
|
||||
expect(calls).toContain(`show:${bPath}`);
|
||||
@@ -940,7 +1093,7 @@ test("POST /sessions uses the catalog default with workspace/thinking settings a
|
||||
],
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const created = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const created = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(created.json()).toEqual({ id: "s1" });
|
||||
expect(sessionNewArg.workspaceConfigPath).toContain(`/snapshots/${"e".repeat(40)}/w.yaml`);
|
||||
expect(sessionNewArg.provider).toBe("zai");
|
||||
@@ -1001,11 +1154,11 @@ test("POST /sessions stops the user's previous open Pi runtime before creating a
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { question: "one" } })).statusCode)
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "one" } })).statusCode)
|
||||
.toBe(200);
|
||||
order.length = 0;
|
||||
|
||||
const second = await app.inject({ method: "POST", url: "/sessions", payload: { question: "two" } });
|
||||
const second = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "two" } });
|
||||
|
||||
expect(second.statusCode).toBe(200);
|
||||
expect(second.json()).toEqual({ id: "s2" });
|
||||
@@ -1026,7 +1179,7 @@ test("POST /sessions refuses to create a session when the local DWH precheck fai
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toMatchObject({ code: "dwh_unreachable" });
|
||||
expect(pinged).toBe(1);
|
||||
@@ -1047,7 +1200,7 @@ test("POST /sessions proceeds past a passing DWH precheck", async () => {
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(pinged).toBe(1);
|
||||
expect(created).toBe(1);
|
||||
@@ -1066,7 +1219,7 @@ test("POST /sessions skips the DWH precheck when the flag is off (default)", asy
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(pinged).toBe(0); // probe never runs without the flag
|
||||
});
|
||||
@@ -1100,7 +1253,7 @@ test("POST /sessions configura Pi con il thinking globale selezionato", async ()
|
||||
],
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(configured.thinking).toBe("high");
|
||||
@@ -1179,6 +1332,29 @@ test("POST /sessions/:id/resume uses the manifest's retained workspace revision"
|
||||
);
|
||||
});
|
||||
|
||||
test.each([undefined, { provider: "local", model: "qwen" }])("resume uses the global model/default, not the historical manifest (%j)", async (payload) => {
|
||||
const configure = vi.fn(async () => {});
|
||||
const manifest = { status: "open", archived: false, provider: "retired", model: "old-model" };
|
||||
const runtime = { bridge: { onClientEvent: () => {}, emitClientEvent: () => {} } };
|
||||
let current: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => current,
|
||||
createFor: () => { current = runtime; return runtime; },
|
||||
configure, start: () => {},
|
||||
} as any,
|
||||
thtRunner: { sessionShow: async () => manifest, reopenSession: async () => {} } as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
runtimeModelCatalog: sessionCatalog("zai/glm-5.2", ["zai/glm-5.2", "local/qwen"]),
|
||||
getSettings: () => ({ workspace: "psd", thinking: "medium" }) as any,
|
||||
});
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/model-resume/resume", ...(payload ? { payload } : {}) });
|
||||
expect(response.statusCode).toBe(200);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(configure).toHaveBeenCalledWith(expect.anything(), expect.objectContaining(payload ?? { provider: "zai", model: "glm-5.2" }));
|
||||
expect(manifest).toMatchObject({ provider: "retired", model: "old-model" });
|
||||
});
|
||||
|
||||
test("POST /sessions/:id/resume returns a sanitized error when its retained revision is unavailable", async () => {
|
||||
const rawFailure = "cannot read /data/workspace-registry/snapshots/secret-revision";
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
@@ -1405,7 +1581,7 @@ test("resuming a different session stops the user's previous Pi runtime", async
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { question: "one" } })).statusCode)
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "one" } })).statusCode)
|
||||
.toBe(200);
|
||||
|
||||
const resumed = await app.inject({ method: "POST", url: "/sessions/s2/resume" });
|
||||
@@ -1793,13 +1969,13 @@ test.each([
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "old" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "old" } });
|
||||
if (lifecycle === "close") {
|
||||
await app.inject({ method: "POST", url: "/sessions/s1/close" });
|
||||
await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
} else {
|
||||
await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "replacement" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "replacement" } });
|
||||
}
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(current).toBe(replacement);
|
||||
@@ -1863,7 +2039,7 @@ test.each(["resolve", "reject"] as const)(
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "old" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "old" } });
|
||||
await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
published.length = 0;
|
||||
|
||||
@@ -2055,7 +2231,7 @@ test("Close suppresses a bootstrap that settles while close persistence is pendi
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
const closeResponse = app.inject({ method: "POST", url: "/sessions/s1/close" });
|
||||
await closeStarted;
|
||||
published.length = 0;
|
||||
@@ -2129,7 +2305,7 @@ test("bootstrap failure persists once and keeps Resume serialized behind that pe
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
oldConfigure.reject(new Error("configure failed"));
|
||||
await failureStarted;
|
||||
const resumeResponse = app.inject({ method: "POST", url: "/sessions/s1/resume" })
|
||||
@@ -2252,7 +2428,7 @@ test("a replaced runtime cannot publish or fail the newly resumed session", asyn
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
oldBridge.setState("idle");
|
||||
|
||||
@@ -2312,7 +2488,7 @@ test("a deleted runtime cannot repopulate or fail the forgotten session", async
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
const response = await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
@@ -2357,7 +2533,7 @@ test("an unexpectedly exited runtime publishes its terminal sequence then releas
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
published.length = 0;
|
||||
|
||||
@@ -2408,7 +2584,7 @@ test("agent_end releases the Pi runtime after the session was finalized", async
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
|
||||
@@ -2480,7 +2656,7 @@ test("POST /sessions/:id/response senza gate pendente risponde 409 (risposta sta
|
||||
getSettings: () => ({ workspace: "w" }),
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
// The fake Pi never emitted a ui_request: the bridge has no pending descriptor, so a
|
||||
// response (stale UI, double submit) must be rejected instead of forwarded to Pi.
|
||||
const res = await app.inject({ method: "POST", url: "/sessions/s1/response",
|
||||
@@ -2621,7 +2797,7 @@ test("POST /sessions readiness failure returns one fixed public message without
|
||||
getSettings: () => ({ workspace: "psd" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toEqual({
|
||||
error: "Session services are not ready. Check configuration and connectivity, then try again.",
|
||||
@@ -2642,7 +2818,7 @@ test.each(["semantic_index_incompatible", "workspace_not_activatable"] as const)
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "q" },
|
||||
method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
@@ -2665,7 +2841,7 @@ test("POST /sessions returns storage 503 before creating a Pi runtime when sessi
|
||||
mgr: { createFor: () => { piCreated = true; throw new Error("must not spawn"); } } as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.json()).toEqual({ error: "session storage is unavailable" });
|
||||
@@ -2685,13 +2861,13 @@ test("POST /sessions proceeds when ollamaEnsure succeeds", async () => {
|
||||
getSettings: () => ({ workspace: "psd" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.json()).toEqual({ id: "s1" });
|
||||
expect(qdrantEnsure).toHaveBeenCalledWith(operationalWorkspace("psd"), 60, "self_heal");
|
||||
expect(ensureWs).toContain(`/snapshots/${"e".repeat(40)}/psd.yaml`);
|
||||
});
|
||||
|
||||
test("POST /sessions falls back from a stale requested model to the catalog default", async () => {
|
||||
test("POST /sessions rejects a stale requested model without silently using the default", async () => {
|
||||
let created = 0;
|
||||
let persisted: any;
|
||||
const runtime = { bridge: { onClientEvent: () => {} } };
|
||||
@@ -2718,16 +2894,13 @@ test("POST /sessions falls back from a stale requested model to the catalog defa
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "q", provider: "deepseek", model: "deepseek-v4-pro" },
|
||||
payload: { interactionLanguage: "en", question: "q", provider: "deepseek", model: "deepseek-v4-pro" },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json()).toEqual({
|
||||
id: "fallback",
|
||||
warning: "Configured model deepseek/deepseek-v4-pro is unavailable; using zai/glm-5.2.",
|
||||
});
|
||||
expect(persisted).toMatchObject({ provider: "zai", model: "glm-5.2" });
|
||||
expect(created).toBe(1);
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toMatchObject({ code: "model_unavailable" });
|
||||
expect(persisted).toBeUndefined();
|
||||
expect(created).toBe(0);
|
||||
});
|
||||
|
||||
test("POST /sessions marks a persisted session failed when runtime construction throws", async () => {
|
||||
@@ -2756,7 +2929,7 @@ test("POST /sessions marks a persisted session failed when runtime construction
|
||||
],
|
||||
});
|
||||
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toEqual({
|
||||
@@ -2849,7 +3022,7 @@ test.each([
|
||||
|
||||
try {
|
||||
const response = flow === "new"
|
||||
? await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } })
|
||||
? await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } })
|
||||
: await app.inject({ method: "POST", url: `/sessions/${sessionId}/resume` });
|
||||
const logs = consoleError.mock.calls.flat().map(String).join(" ");
|
||||
|
||||
@@ -2955,7 +3128,7 @@ test("POST /sessions returns after bridge attachment but starts only after retri
|
||||
getSettings: () => ({ workspace: "psd" }) as any,
|
||||
});
|
||||
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.json()).toEqual({ id: "s-early" });
|
||||
expect(bridgeAttached).toBe(true);
|
||||
expect(started).toBe(false);
|
||||
@@ -3002,7 +3175,7 @@ test("POST /sessions bootstrap failure emits only a fixed recovery message", asy
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "q" },
|
||||
payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
@@ -3098,7 +3271,7 @@ test.each([
|
||||
})),
|
||||
} as any,
|
||||
});
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(expectedStatus);
|
||||
expect(ensure).toHaveBeenCalledTimes(reachesReadiness ? 1 : 0);
|
||||
|
||||
@@ -160,8 +160,7 @@ test("GET /models returns session choices from the installation model catalog",
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {} as any,
|
||||
runtimeModelCatalog: {
|
||||
defaultSession: "zai/glm-5.2",
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: "zai/glm-5.2",
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
sessionModels: () => [{
|
||||
id: "zai/glm-5.2", provider: "zai", model: "glm-5.2", label: "GLM 5.2",
|
||||
@@ -187,8 +186,7 @@ test("GET /models returns an empty list when the catalog has no session models",
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {} as any,
|
||||
runtimeModelCatalog: {
|
||||
defaultSession: null,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: null,
|
||||
embedding: null,
|
||||
sessionModels: () => [],
|
||||
metadataModels: () => [],
|
||||
|
||||
@@ -34,6 +34,19 @@ test("sessionNew parses id from JSON", async () => {
|
||||
expect(await r.sessionNew({ question: "q" })).toEqual({ id: "2026-06-27-100000-x" });
|
||||
});
|
||||
|
||||
test("session language uses public per-command CLI flags and the selected config", async () => {
|
||||
const runner = new ThtRunner({ thtBin: "tht", harnessDir: "/h", configPath: "config/tht.yaml" });
|
||||
(spawn as any).mockClear();
|
||||
await runner.sessionNew({ question: "Pazienti", interactionLanguage: "en" });
|
||||
expect((spawn as any).mock.calls[0][1]).toEqual([
|
||||
"session", "new", "Pazienti", "--interaction-language", "en", "--json", "-c", "config/tht.yaml",
|
||||
]);
|
||||
await runner.ensureInteractionLanguage("s1");
|
||||
expect((spawn as any).mock.calls[1][1]).toEqual([
|
||||
"session", "ensure-interaction-language", "s1", "--json", "-c", "config/tht.yaml",
|
||||
]);
|
||||
});
|
||||
|
||||
test("searchPack persists retrieval context with session and workspace", async () => {
|
||||
const calls: any[] = [];
|
||||
const r = new ThtRunner({ thtBin: "tht", harnessDir: "/h", configPath: "config/tht.yaml" });
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, symlinkSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { parseWorkspaceCatalogYaml } from "../src/workspaces/catalog.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
function directory() {
|
||||
const root = mkdtempSync(join(tmpdir(), "tht-documents-"));
|
||||
roots.push(root);
|
||||
return join(root, "workspaces");
|
||||
}
|
||||
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
|
||||
|
||||
function cli(...args: string[]) {
|
||||
const packaged = process.env.THT_WORKSPACE_TEST_CLI;
|
||||
const result = spawnSync(packaged ?? process.execPath, [...(packaged ? ["workspace"] : ["--import", "tsx", resolve("src/workspace-documents-cli.ts")]), ...args, "--json"], {
|
||||
encoding: "utf8", env: { ...process.env, PATH: "" },
|
||||
});
|
||||
return { ...result, report: result.stdout.trim() ? JSON.parse(result.stdout) : null };
|
||||
}
|
||||
|
||||
const descriptor = "workspace:\n schema_version: 4\n id: practice\n name: Practice\n language: en\n";
|
||||
const catalog = "schema_version: 1\nworkspaces:\n - id: practice\n name: Practice\n";
|
||||
const corpus = [
|
||||
{ name: "minimal", descriptor, catalog, valid: true },
|
||||
{ name: "optional Evidence", descriptor: descriptor + "evidence:\n source:\n type: http\n uris: [https://example.com/manual.md]\n", catalog, valid: true },
|
||||
{ name: "duplicate YAML key", descriptor: descriptor + " id: practice\n", catalog, valid: false },
|
||||
{ name: "ambiguous document", descriptor: descriptor + "---\n" + descriptor, catalog, valid: false },
|
||||
{ name: "unknown workspace field", descriptor: descriptor + " secret: VERY_SECRET_VALUE\n", catalog, valid: false },
|
||||
{ name: "database binding in authored descriptor", descriptor: descriptor + "dwh: {password: VERY_SECRET_VALUE}\n", catalog, valid: false },
|
||||
{ name: "duplicate catalog id", descriptor, catalog: catalog + " - id: practice\n name: Practice\n", valid: false },
|
||||
{ name: "unknown catalog field", descriptor, catalog: catalog + "secret: VERY_SECRET_VALUE\n", valid: false },
|
||||
{ name: "invalid catalog YAML", descriptor, catalog: catalog + "schema_version: 1\n", valid: false },
|
||||
{ name: "unsafe Evidence URI", descriptor: descriptor + "evidence:\n source:\n type: http\n uris: [https://user:VERY_SECRET_VALUE@example.com/file]\n", catalog, valid: false },
|
||||
];
|
||||
|
||||
test.each(corpus)("CLI and runtime agree: $name", (fixture) => {
|
||||
const root = directory();
|
||||
mkdirSync(join(root, "practice"), { recursive: true });
|
||||
writeFileSync(join(root, "thoth-workspaces.yaml"), fixture.catalog);
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), fixture.descriptor);
|
||||
let accepted = true;
|
||||
try { parseWorkspaceCatalogYaml(fixture.catalog); parseWorkspaceYaml(fixture.descriptor); }
|
||||
catch { accepted = false; }
|
||||
expect(accepted).toBe(fixture.valid);
|
||||
const checked = cli("validate", "--directory", root);
|
||||
expect(checked.status).toBe(fixture.valid ? 0 : 1);
|
||||
expect(checked.report.ok).toBe(accepted);
|
||||
expect(checked.stdout + checked.stderr).not.toContain("VERY_SECRET_VALUE");
|
||||
if (!fixture.valid) {
|
||||
expect(checked.report.issues[0].document).not.toBe(".");
|
||||
expect(checked.report.issues[0].correction.length).toBeGreaterThan(10);
|
||||
}
|
||||
});
|
||||
|
||||
test("prepare refuses existing directories and invalid options without changing documents", () => {
|
||||
const root = directory();
|
||||
expect(cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice").status).toBe(0);
|
||||
const before = readFileSync(join(root, "thoth-workspaces.yaml"), "utf8");
|
||||
expect(cli("prepare", "--directory", root, "--id", "other", "--name", "Other").report.issues[0].code).toBe("destination_exists");
|
||||
expect(readFileSync(join(root, "thoth-workspaces.yaml"), "utf8")).toBe(before);
|
||||
expect(cli("validate", "--directory", root, "--typo", "VERY_SECRET_VALUE").status).toBe(2);
|
||||
});
|
||||
|
||||
test("validation rejects directory mismatches, missing references and symlinks without following them", () => {
|
||||
const root = directory();
|
||||
cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice");
|
||||
mkdirSync(join(root, "unlisted"));
|
||||
expect(cli("validate", "--directory", root).report.issues.some((i: {code: string}) => i.code === "unlisted_directory")).toBe(true);
|
||||
rmSync(join(root, "unlisted"), { recursive: true });
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), descriptor.replace("name: Practice", "name: Different"));
|
||||
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("catalog_mismatch");
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), descriptor + "evidence:\n source:\n type: filesystem\n uri: practice/evidence\n");
|
||||
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("missing_reference");
|
||||
symlinkSync(roots[roots.length - 1], join(root, "practice/evidence"), "dir");
|
||||
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("unsafe_reference");
|
||||
});
|
||||
|
||||
test("local Evidence checks references and YAML frontmatter, and explicitly defers canonical content validation", () => {
|
||||
const root = directory();
|
||||
cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice");
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), descriptor + "evidence:\n schema_version: 2\n source:\n type: filesystem\n uri: practice/evidence\n patterns: ['curated/**/*.md']\n");
|
||||
mkdirSync(join(root, "practice/evidence/curated/domain"), { recursive: true });
|
||||
const evidence = join(root, "practice/evidence/curated/domain/rule.md");
|
||||
writeFileSync(evidence, "---\nschema_version: 4\nid: evidence:rule\nkind: domain\nlanguage: en\npurposes: [sql_generation]\n---\n# Rule\n\n## Rule\nUse the order number.\n");
|
||||
const checked = cli("validate", "--directory", root);
|
||||
expect(checked.status).toBe(0);
|
||||
expect(checked.report.workspaces).toEqual([{ id: "practice", evidence: "local-files" }]);
|
||||
expect(checked.report.deferred_checks).toContain("practice:evidence-content-provenance-and-indexing");
|
||||
writeFileSync(evidence, "---\nid: first\nid: VERY_SECRET_VALUE\n---\n# Rule\n");
|
||||
const invalid = cli("validate", "--directory", root);
|
||||
expect(invalid.status).toBe(1);
|
||||
expect(invalid.report.issues[0].document).toBe("practice/evidence/curated/domain/rule.md");
|
||||
expect(invalid.stdout).not.toContain("VERY_SECRET_VALUE");
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), descriptor + "evidence:\n source:\n type: filesystem\n uri: practice/evidence\n patterns: [missing.md]\n");
|
||||
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("missing_reference");
|
||||
});
|
||||
|
||||
test("prepare creates documents accepted by the runtime and validate is repeatable without installation or services", () => {
|
||||
const root = directory();
|
||||
const prepared = cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice");
|
||||
expect(prepared.stderr).toBe("");
|
||||
expect(prepared.status).toBe(0);
|
||||
expect(prepared.report.ok).toBe(true);
|
||||
const catalog = readFileSync(join(root, "thoth-workspaces.yaml"), "utf8");
|
||||
const descriptor = readFileSync(join(root, "practice/workspace.yaml"), "utf8");
|
||||
expect(parseWorkspaceCatalogYaml(catalog).workspaces[0].id).toBe("practice");
|
||||
expect(parseWorkspaceYaml(descriptor).evidence).toBeUndefined();
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const checked = cli("validate", "--directory", root);
|
||||
expect(checked.status).toBe(0);
|
||||
expect(checked.report.workspaces).toEqual([{ id: "practice", evidence: "absent" }]);
|
||||
}
|
||||
expect(readFileSync(join(root, "thoth-workspaces.yaml"), "utf8")).toBe(catalog);
|
||||
expect(readFileSync(join(root, "practice/workspace.yaml"), "utf8")).toBe(descriptor);
|
||||
});
|
||||
@@ -10,6 +10,28 @@ import {
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
test("Evidence consolidation runs only its stage and preserves blocked Catalog readiness", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", counts: { documents: 2 } }), stderr: "" }));
|
||||
const result = await service({ repository: catalog, runChild }).consolidateEvidence({ workspaceId: "catalog-workspace" });
|
||||
expect(result.status).toBe("succeeded");
|
||||
expect(runChild).toHaveBeenCalledOnce();
|
||||
expect(runChild.mock.calls[0]?.[0]).toMatchObject({ argv: ["preprocess", "evidence", "--consolidate", "--json", "-c", "/dev/fd/3"] });
|
||||
expect(catalog.beginPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.clearPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("Evidence index failure reports retry without changing Catalog state", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({ exitCode: 1, stdout: JSON.stringify({ status: "failed", saved: true, error: "Saved; retry indexing." }), stderr: "private provider endpoint" }));
|
||||
const result = await service({ repository: catalog, runChild }).consolidateEvidence({ workspaceId: "catalog-workspace" });
|
||||
expect(result.status).toBe("failed");
|
||||
expect(result.warnings).toEqual(["Saved; retry indexing."]);
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(result)).not.toContain("private provider");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
|
||||
});
|
||||
@@ -254,6 +276,24 @@ test("semantic preflight failure is persisted before returning", async () => {
|
||||
);
|
||||
});
|
||||
|
||||
test.each(["refresh", "decide"] as const)("Evidence %s calls only the source worker and preserves Catalog readiness", async action => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({exitCode: 0, stdout: JSON.stringify({status: "succeeded", counts: {changed: 1}}), stderr: ""}));
|
||||
const result = await service({repository: catalog, runChild}).evidenceSources({workspaceId: "catalog-workspace", action,
|
||||
...(action === "decide" ? {sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "replace" as const} : {}), actor: "Curator"});
|
||||
expect(result.status).toBe("succeeded");
|
||||
expect(runChild).toHaveBeenCalledWith({configPath: expect.any(String), argv: expect.arrayContaining(["evidence", "sources", action, "-c", "/dev/fd/3", "--actor", "Curator"])});
|
||||
expect(catalog.beginPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.clearPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("failed source activation reports the saved decision for retry", async () => {
|
||||
const result = await service({runChild: async () => ({exitCode: 1, stdout: JSON.stringify({status: "failed", saved: true, error: "Decision saved; retry indexing"}), stderr: "secret"})})
|
||||
.evidenceSources({workspaceId: "catalog-workspace", action: "decide", sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "keep"});
|
||||
expect(result).toMatchObject({status: "failed", warnings: ["Decision saved; retry indexing"]});
|
||||
});
|
||||
|
||||
test("clear invalidates Catalog readiness before clearing only derived worker data", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
@@ -202,7 +203,7 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
const suffix = first.inputFingerprint.slice(7, 23);
|
||||
const suffix = createHash("sha256").update(first.inputFingerprint + "\n" + readFileSync(first.path, "utf8")).digest("hex").slice(0, 16);
|
||||
expect(second.path).toBe(first.path);
|
||||
expect(first.path).toBe(join(
|
||||
f.dataRoot,
|
||||
|
||||
@@ -242,6 +242,7 @@ test("separate runtime leases hand off byte-identical revision Evidence configs
|
||||
source_identity: "workspace://psd-clinical",
|
||||
});
|
||||
expect(parse(firstYaml).evidence).toEqual({
|
||||
local_archive_root: join(f.registryConfig.root, "repo", "psd-clinical"),
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: expectedRoot,
|
||||
|
||||
@@ -192,6 +192,7 @@ test("renders filesystem Evidence below the immutable revision content root with
|
||||
expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision);
|
||||
expect(rendered.evidence).toEqual({
|
||||
schema_version: 2,
|
||||
local_archive_root: "/srv/registry/repo/psd-clinical",
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`,
|
||||
@@ -203,7 +204,7 @@ test("renders filesystem Evidence below the immutable revision content root with
|
||||
max_chunk_chars: 4_000,
|
||||
retain_published_generations: 3,
|
||||
});
|
||||
expect(yaml).not.toContain("/srv/registry/repo");
|
||||
expect(rendered.evidence.sources[0].root).not.toContain("/srv/registry/repo");
|
||||
});
|
||||
|
||||
test("renders public HTTP Evidence with exact fractional-second timeouts and every policy limit", () => {
|
||||
@@ -223,6 +224,7 @@ test("renders public HTTP Evidence with exact fractional-second timeouts and eve
|
||||
}));
|
||||
|
||||
expect(rendered.evidence).toEqual({
|
||||
local_archive_root: "/srv/registry/repo/psd-clinical",
|
||||
sources: [{
|
||||
type: "http",
|
||||
urls: ["https://evidence.example.test/guide.md"],
|
||||
|
||||
+3
-2
@@ -13,6 +13,7 @@ services:
|
||||
THT_BIN: /opt/venv/bin/tht
|
||||
THT_DATA_ROOT: /data
|
||||
SETTINGS_FILE: /data/settings/settings.json
|
||||
THT_EVIDENCE_HOST_REGISTRY_ROOT: ${THT_WORKSPACE_REGISTRY_ROOT:-}
|
||||
THT_MAINTENANCE_FILE: /data/settings/maintenance.json
|
||||
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
|
||||
THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE}
|
||||
@@ -99,7 +100,7 @@ services:
|
||||
image: thothii-core:local
|
||||
profiles: [catalog-maintenance]
|
||||
pull_policy: never
|
||||
command: ["node", "/app/backend/dist/catalog/migrate.js"]
|
||||
command: ["bash", "/app/docker/catalog-migrate.sh"]
|
||||
environment:
|
||||
THT_CATALOG_DB_HOST: catalog-db
|
||||
THT_CATALOG_DB_PORT: "5432"
|
||||
@@ -153,7 +154,7 @@ services:
|
||||
- type: volume
|
||||
source: workspace-registry
|
||||
target: /data/workspace-registry
|
||||
read_only: true
|
||||
read_only: false
|
||||
- type: volume
|
||||
source: sessions
|
||||
target: /data/sessions
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Optional local profile: expose the existing curator checkout on the installation host.
|
||||
# Copy the existing registry repo to this path before enabling the override. Snapshot/state
|
||||
# volumes remain unchanged. THT_EVIDENCE_HOST_REGISTRY_ROOT must be an absolute host path.
|
||||
services:
|
||||
core:
|
||||
environment:
|
||||
THT_EVIDENCE_HOST_REGISTRY_ROOT: ${THT_EVIDENCE_HOST_REGISTRY_ROOT:?set an absolute host registry path}
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${THT_EVIDENCE_HOST_REGISTRY_ROOT:?set an absolute host registry path}/repo
|
||||
target: /data/workspace-registry/repo
|
||||
bind:
|
||||
create_host_path: false
|
||||
workspace-maintenance:
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${THT_EVIDENCE_HOST_REGISTRY_ROOT:?set an absolute host registry path}/repo
|
||||
target: /data/workspace-registry/repo
|
||||
bind:
|
||||
create_host_path: false
|
||||
@@ -45,7 +45,7 @@ services:
|
||||
- type: bind
|
||||
source: ${THT_WORKSPACE_REGISTRY_ROOT:?set THT_WORKSPACE_REGISTRY_ROOT}
|
||||
target: /data/workspace-registry
|
||||
read_only: true
|
||||
read_only: false
|
||||
- type: bind
|
||||
source: ${THT_DATA_ROOT:?set THT_DATA_ROOT}/workspace-secrets
|
||||
target: /data/workspace-secrets
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# The image is tagged from the running frontend before the visual review is published.
|
||||
services:
|
||||
frontend:
|
||||
image: thothii-frontend:before-visual-review-20260912
|
||||
@@ -0,0 +1,8 @@
|
||||
# Local-only review override. Apply after the existing local installation profiles.
|
||||
# Changes only the frontend image; Core, configuration and volumes remain unchanged.
|
||||
services:
|
||||
frontend:
|
||||
image: thothii-frontend:visual-review-20260912
|
||||
build:
|
||||
context: /Users/mp/projects/ThothII-visual-review
|
||||
dockerfile: docker/frontend.Dockerfile
|
||||
@@ -0,0 +1,5 @@
|
||||
# Local-only rollback to the full-shell frontend before visual integration.
|
||||
# Apply after local installation profiles, with --no-build and --no-deps.
|
||||
services:
|
||||
frontend:
|
||||
image: thothii-frontend:before-visual-shell-merge-20260913
|
||||
@@ -0,0 +1,17 @@
|
||||
services:
|
||||
core:
|
||||
environment:
|
||||
THT_EVIDENCE_HOST_REGISTRY_ROOT: /Users/mp/projects/ThothII/deploy/psd/evidence-registry
|
||||
volumes:
|
||||
- type: bind
|
||||
source: /Users/mp/projects/ThothII/deploy/psd/evidence-registry/repo
|
||||
target: /data/workspace-registry/repo
|
||||
bind:
|
||||
create_host_path: false
|
||||
workspace-maintenance:
|
||||
volumes:
|
||||
- type: bind
|
||||
source: /Users/mp/projects/ThothII/deploy/psd/evidence-registry/repo
|
||||
target: /data/workspace-registry/repo
|
||||
bind:
|
||||
create_host_path: false
|
||||
Submodule
+1
Submodule deploy/psd/evidence-registry/repo added at 24fb53236b
@@ -2,6 +2,10 @@
|
||||
# Replace every absolute path before using this as an advanced reference.
|
||||
schemaVersion: 2
|
||||
profile: local
|
||||
# Mac standalone example; the Omics server requires embedded/upstream separately.
|
||||
shell:
|
||||
mode: full
|
||||
defaultLocale: en
|
||||
projectDirectory: "<abs>/projects/ThothII"
|
||||
envFile: "<abs>/projects/ThothII/deploy/psd/operator.env"
|
||||
workspaceRepository:
|
||||
@@ -10,22 +14,28 @@ workspaceRepository:
|
||||
access: ssh
|
||||
modelCatalog:
|
||||
defaults:
|
||||
session: zai/glm-5.3
|
||||
metadataGeneration: zai/glm-5.3
|
||||
interaction: zai/glm-5.3
|
||||
embedding:
|
||||
id: ollama/qwen3-embedding:0.6b
|
||||
dimensions: 1024
|
||||
providers:
|
||||
deepseek:
|
||||
authentication:
|
||||
mode: pi_auth
|
||||
mode: secret_env
|
||||
apiKeyEnv: DEEPSEEK_API_KEY
|
||||
session:
|
||||
mode: pi_builtin
|
||||
metadataGeneration:
|
||||
litellmProvider: deepseek
|
||||
models:
|
||||
deepseek-v4-pro:
|
||||
label: DeepSeek V4 Pro
|
||||
session: {}
|
||||
metadataGeneration: {}
|
||||
deepseek-v4-flash:
|
||||
label: DeepSeek V4 Flash
|
||||
session: {}
|
||||
metadataGeneration: {}
|
||||
zai:
|
||||
endpoint:
|
||||
baseUrl: https://api.z.ai/api/coding/paas/v4
|
||||
|
||||
@@ -8,6 +8,11 @@ cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets
|
||||
chmod 600 deploy/secrets/thothii.secrets
|
||||
```
|
||||
|
||||
For a normal local installation, `tht setup --complete` creates the active bundle at
|
||||
`deploy/local/secrets/thothii.secrets` and creates the two Catalog password files beside it. The
|
||||
generated `deploy/local/operator.env` contains only absolute paths to those files; never copy
|
||||
secret values into `operator.env`.
|
||||
|
||||
The file uses strict `KEY=VALUE` lines (comments and blank lines are allowed). The supported
|
||||
installation keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_CA`, `THT_SSL_CA`,
|
||||
`THT_OIDC_CLIENT_SECRET`, and `THT_AUTHENTIK_API_TOKEN`. Installation Model Catalog providers may
|
||||
@@ -29,6 +34,11 @@ Session and metadata-generation runtimes read only the provider key named by
|
||||
credential reference, not their execution lifecycle. Pi-owned authentication remains available only
|
||||
to session-only built-in providers through `authentication.mode: pi_auth`.
|
||||
|
||||
Workspace database credentials are intentionally not part of this global bundle. Configure each
|
||||
workspace's database binding, password/token, tunnel key and CA in Database Management; the
|
||||
installation stores those values in its encrypted workspace secret store. The workspace Git
|
||||
repository may declare database identity and Evidence, but must never contain these credentials.
|
||||
|
||||
Do not add vector or embedding endpoint credentials to the bundle. Active operator manuals use
|
||||
internal Qdrant and Ollama services, so vector/embedding runtime endpoint secrets are not part of
|
||||
the supported installation contract.
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
node /app/backend/dist/catalog/migrate.js
|
||||
/opt/venv/bin/python -m tht.memory.migrate
|
||||
@@ -131,7 +131,7 @@ ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH" \
|
||||
HOME=/home/thoth
|
||||
|
||||
COPY scripts/verify-line-endings.sh /usr/local/bin/verify-line-endings
|
||||
COPY docker/core-entrypoint.sh docker/workspace-maintenance-entrypoint.sh docker/session-migrate.sh docker/ensure-pi-trust.mjs docker/embedding-model-init.sh /app/docker/
|
||||
COPY docker/core-entrypoint.sh docker/workspace-maintenance-entrypoint.sh docker/session-migrate.sh docker/catalog-migrate.sh docker/ensure-pi-trust.mjs docker/embedding-model-init.sh /app/docker/
|
||||
COPY docker/smoke/core-smoke.sh /app/docker/smoke/core-smoke.sh
|
||||
RUN /usr/local/bin/verify-line-endings /app/docker \
|
||||
&& chmod +x /app/docker/core-entrypoint.sh /app/docker/workspace-maintenance-entrypoint.sh /app/docker/session-migrate.sh /app/docker/embedding-model-init.sh /app/docker/smoke/core-smoke.sh
|
||||
|
||||
@@ -1,6 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
test "$(cat /usr/share/nginx/html/config.js)" = 'window.__THOTHII_CONFIG__ = {};'
|
||||
# The generic image contains an empty fallback; installed containers mount the generated
|
||||
# public projection over it. An optional path lets host projection tests use this same check.
|
||||
config_file=${1:-/usr/share/nginx/html/config.js}
|
||||
test -f "$config_file" && test -r "$config_file"
|
||||
config=$(tr -d '[:space:]' < "$config_file")
|
||||
case "$config" in
|
||||
'window.__THOTHII_CONFIG__={};')
|
||||
test -z "${THT_FRONTEND_CONFIG_REVISION:-}"
|
||||
;;
|
||||
*)
|
||||
# Installation Load validates BCP47 syntax. Here check only the public payload shape;
|
||||
# catalog availability and fallback belong to the frontend, not the generic image.
|
||||
locale='[A-Za-z][A-Za-z0-9]*(-[A-Za-z0-9]+)*'
|
||||
full="\"mode\":\"full\",\"defaultLocale\":\"$locale\""
|
||||
embedded="\"mode\":\"embedded\",\"defaultLocale\":\"$locale\",\"adapter\":\"omics-portal\""
|
||||
if ! printf '%s\n' "$config" | LC_ALL=C grep -Eq "^window\.__THOTHII_CONFIG__=\{\"backendBaseUrl\":\"/api\",\"shell\":\{($full|$embedded)\}\};$"; then
|
||||
echo "Invalid frontend public runtime configuration" >&2
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
printf '%s\n' "frontend runtime config smoke: ok"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user