Compare commits
57
Commits
development
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f53512e4d | ||
|
|
497ab84031 | ||
|
|
0d2e573e0d | ||
|
|
bd416f7327 | ||
|
|
23e52c80de | ||
|
|
84084bba37 | ||
|
|
efd7d788d9 | ||
|
|
b1c510a097 | ||
|
|
5f3680a0fb | ||
|
|
4ff91e8d6e | ||
|
|
5f3a7f5975 | ||
|
|
043ffdfad6 | ||
|
|
6a4634dcf1 | ||
|
|
84804be9f8 | ||
|
|
c3caba94dd | ||
|
|
b1723c34c4 | ||
|
|
d6cdffea62 | ||
|
|
49333a2d35 | ||
|
|
b006b94479 | ||
|
|
bdcd8fcd28 | ||
|
|
cf90c1bd51 | ||
|
|
571a4bcaa2 | ||
|
|
9051463654 | ||
|
|
26c5605ff7 | ||
|
|
3535fda958 | ||
|
|
2953f6b608 | ||
|
|
45db3a239b | ||
|
|
7d826e46c0 | ||
|
|
648434a32e | ||
|
|
023b822f83 | ||
|
|
d8a29bfbdd | ||
|
|
a59624a68f | ||
|
|
5af4408194 | ||
|
|
e088abd60a | ||
|
|
803e9e9201 | ||
|
|
8c81996896 | ||
|
|
eed398e569 | ||
|
|
c8d276ddc6 | ||
|
|
2d1b714ebe | ||
|
|
c7e5f295e6 | ||
|
|
f52bf22e05 | ||
|
|
840344706f | ||
|
|
41b9fed4d5 | ||
|
|
36bf659ea9 | ||
|
|
4a67d60233 | ||
|
|
debb63d87b | ||
|
|
3943022a97 | ||
|
|
f5ec2d9313 | ||
|
|
82e2c91f42 | ||
|
|
8fe526dd6e | ||
|
|
e68e80a33d | ||
|
|
818563c408 | ||
|
|
50c546e42d | ||
|
|
651a5c7902 | ||
|
|
28db30bd78 | ||
|
|
cffa60772e | ||
|
|
8707ae1d46 |
@@ -17,6 +17,7 @@ frontend/vite.database-management-prototype.config.ts
|
||||
!deploy/env/*.env.example
|
||||
deploy/thothii.env
|
||||
deploy/secrets/
|
||||
deploy/psd/
|
||||
harness/workspaces/*.yaml
|
||||
!harness/workspaces/local.yaml
|
||||
!harness/workspaces/tht.example.yaml
|
||||
@@ -29,3 +30,7 @@ coverage/
|
||||
data/
|
||||
sessions/
|
||||
workspace-registry/
|
||||
|
||||
.tht/
|
||||
|
||||
deploy/local/
|
||||
|
||||
@@ -7,6 +7,11 @@ on:
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "mkdocs.yml"
|
||||
- "scripts/build-docs.sh"
|
||||
- "scripts/verify-public-docs.py"
|
||||
- "scripts/test-verify-public-docs.py"
|
||||
- "scripts/verify-auth-docs.py"
|
||||
- "scripts/test-verify-auth-docs.py"
|
||||
- "docs/requirements.txt"
|
||||
- ".gitea/workflows/publish-docs.yml"
|
||||
workflow_dispatch:
|
||||
@@ -34,14 +39,25 @@ jobs:
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: docs/requirements.txt
|
||||
cache-dependency-path: docs/requirements.lock
|
||||
|
||||
- name: Install MkDocs dependencies
|
||||
run: python -m pip install -r docs/requirements.txt
|
||||
run: python -m pip install -r docs/requirements.lock
|
||||
|
||||
- name: Test public documentation boundary
|
||||
run: python scripts/test-verify-public-docs.py
|
||||
|
||||
- name: Test current authentication documentation
|
||||
run: |
|
||||
python scripts/verify-auth-docs.py auth
|
||||
python scripts/verify-auth-docs.py dwh
|
||||
python scripts/test-verify-auth-docs.py auth
|
||||
python scripts/test-verify-auth-docs.py dwh
|
||||
|
||||
- name: Build documentation
|
||||
# Some documented source files intentionally live outside docs/.
|
||||
run: mkdocs build
|
||||
run: |
|
||||
mkdocs build --strict
|
||||
python scripts/verify-public-docs.py
|
||||
|
||||
- name: Publish generated site to the pages branch
|
||||
working-directory: site
|
||||
|
||||
@@ -46,6 +46,7 @@ deploy/secrets/*
|
||||
# Per-installation configuration generated by `tht setup` (examples stay tracked).
|
||||
deploy/*/thothii-installation.yaml
|
||||
deploy/*/operator.env
|
||||
deploy/*/auth/
|
||||
deploy/*/generated/
|
||||
deploy/*/secrets/*
|
||||
!deploy/*/secrets/.gitkeep
|
||||
|
||||
@@ -98,10 +98,21 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
|
||||
- **`tht`'s `-c`/`--config` is a PER-COMMAND option** — it must follow the subcommand, never
|
||||
precede it (`ThtRunner.buildArgv` enforces this; prepending caused live 500s).
|
||||
- **`--json` output must be pristine** (only valid JSON on stdout) — used as a machine contract.
|
||||
- **UI strings are English; document *content* stays the workspace language** (Italian for
|
||||
`psd`) because it's the real data. Only chrome/labels are English.
|
||||
- **Workspace schema v4** defines database and Evidence concerns only. Embedding/model facts come
|
||||
from the installation catalog. The legacy `harness/workspaces/*.yaml` runtime snapshots still use
|
||||
- **Localization:** deterministic UI uses the EN/IT catalogs with English fallback;
|
||||
model interaction uses the session manifest's immutable `interaction_language`.
|
||||
Workspace content, SQL, and identifiers remain unchanged. For shell modes, portal
|
||||
integration, or translations, read `docs/operations/shell-and-localization.md`.
|
||||
- **Server deployment:** for the coordinated ThothII/Omics upgrade, follow
|
||||
`docs/operations/server-codex-handoff.md`; it supersedes earlier Omics delivery
|
||||
instructions. Omics source integration uses GitHub with no repository relay prerequisite.
|
||||
- **Server identity:** for portal login/logout, proxy headers, or Omics deploy, read
|
||||
`docs/install/authentication-upstream.md` before changing authentication. Omics
|
||||
uses embedded/upstream, not a second ThothII OIDC login. Full/embedded rendering
|
||||
is documented in `docs/architecture/application-shell.md`; release acceptance
|
||||
is in `docs/testing/authentication-manual-acceptance.md`.
|
||||
- **Workspace schema v4** defines workspace identity and optional Evidence only. PostgreSQL Metadata
|
||||
Catalog owns database identity, binding, schema, descriptions, sensitivity, and relationships;
|
||||
embedding/model facts come from the installation catalog. The legacy `harness/workspaces/*.yaml` runtime snapshots still use
|
||||
absolute session/artifact/index paths; secrets stay in `harness/.env` (gitignored).
|
||||
- **Settings are global** (`backend/data/settings.json`: workspace/thinking). Provider/model choices
|
||||
are ephemeral canonical catalog selections pinned into the session manifest.
|
||||
|
||||
+185
-28
@@ -91,21 +91,79 @@ correzione successiva crea una nuova sessione derivata, collegata a quella prece
|
||||
dopo la finalizzazione. Non può modificare il ledger, gli artifact canonici o lo stato
|
||||
terminale della sessione.
|
||||
|
||||
## Memory
|
||||
|
||||
**Memory Module** — Il modulo che possiede le conoscenze ed esperienze curate per
|
||||
migliorare schema linking e generazione SQL di domande future. Le Memory appartengono
|
||||
a un workspace e rimangono distinte dalle Evidence.
|
||||
|
||||
**Memory Card** — L'unità di contenuto gestibile del Memory Module, con identità,
|
||||
ambito di applicazione e provenienza. Il formato è allineato per analogia alle
|
||||
Evidence, senza implicare la stessa origine o lo stesso percorso di pubblicazione.
|
||||
|
||||
**Reusable Memory** — Una Memory Card che esprime un chiarimento di dominio, una
|
||||
regola di costruzione SQL o un errore da evitare con motivo compreso e approvato.
|
||||
La sua validità è circoscritta a un ambito esplicito e non deriva dalla sola
|
||||
approvazione di una scelta occasionale in una domanda.
|
||||
|
||||
**Solved Question** — Una Memory Card che conserva una domanda risolta con la
|
||||
relativa soluzione SQL e il contesto necessario a interpretarla. È un exemplar
|
||||
consultativo: i parametri e le scelte del caso non diventano regole generali.
|
||||
|
||||
**Memory Graph** — L'insieme dei collegamenti espliciti fra card che contribuisce
|
||||
al recupero di conoscenze pertinenti oltre alla somiglianza del contenuto. Il
|
||||
ritrovamento di una card tramite un collegamento non ne implica l'approvazione.
|
||||
|
||||
**Memory Link** — Un collegamento curato fra card, con destinazione e significato
|
||||
espliciti, che contribuisce alla consultazione di contenuti pertinenti. La sua
|
||||
rimozione non comporta la cancellazione delle card collegate.
|
||||
|
||||
## Evidence
|
||||
|
||||
**Context specialist** — La persona competente sul dominio che redige e cura il
|
||||
contenuto delle Evidence. Può essere distinta da chi amministra l'installazione;
|
||||
il suo lavoro di redazione non richiede accesso al database applicativo.
|
||||
|
||||
**Evidence draft** — Il documento iniziale scritto dallo specialista di contesto,
|
||||
che il sistema acquisisce e raffina in Evidence Unit. Può essere redatto e
|
||||
consegnato indipendentemente dall'installazione che userà le Evidence risultanti.
|
||||
|
||||
**Evidence Module** — Il modulo autonomo che possiede la preparazione delle Evidence e
|
||||
la loro consultazione durante il workflow. La preparazione avviene fuori dalle singole
|
||||
sessioni; il workflow usa soltanto contenuti già pubblicati. A runtime contribuisce agli
|
||||
stage semantici esistenti, senza diventare uno stage visibile e senza modificare ledger,
|
||||
artifact o stato del workflow.
|
||||
|
||||
**Source Evidence** — Un documento originale del workspace, conservato senza modifiche
|
||||
come riferimento umano e origine della successiva ristrutturazione.
|
||||
**Source Evidence** — Il documento o la dichiarazione che sostiene il contenuto
|
||||
corrente di una Evidence Unit. Un documento acquisito viene conservato come
|
||||
riferimento umano; una dichiarazione manuale attribuisce il contenuto alla persona
|
||||
che lo ha scritto e approvato.
|
||||
|
||||
**Manual Evidence declaration** — Una dichiarazione esplicita dell'amministratore
|
||||
che sostiene una Evidence creata direttamente o una correzione del suo significato.
|
||||
Non implica una verifica indipendente da parte di una fonte documentale esterna.
|
||||
|
||||
**Evidence origin** — Il documento da cui una Evidence Unit è stata inizialmente
|
||||
derivata. Può restare collegato per provenienza e confronto con gli aggiornamenti
|
||||
anche quando una dichiarazione manuale sostiene il testo corrente. La sola origine
|
||||
non dimostra il supporto semantico di una successiva correzione.
|
||||
|
||||
**Local Evidence archive** — L'insieme delle Evidence curate custodite
|
||||
dall'installazione, distinto dalle draft originali e dai contenuti derivati per
|
||||
la ricerca. Comprende le correzioni manuali e i ritiri deliberati.
|
||||
|
||||
**Consolidated Evidence** — Una versione delle Evidence locali controllata come
|
||||
insieme coerente e pronta per l'attivazione. I file ancora in modifica non ne
|
||||
cambiano il contenuto.
|
||||
|
||||
**Active Evidence** — La versione consolidata disponibile alla consultazione del
|
||||
core. Un tentativo di aggiornamento fallito conserva la versione attiva precedente.
|
||||
|
||||
**Evidence Unit** — La più piccola unità semantica coerente, revisionabile e ricercabile
|
||||
derivata da una sola Source Evidence. Possiede un identificatore stabile indipendente
|
||||
dal kind, assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono
|
||||
fuse automaticamente.
|
||||
fondata su una Source Evidence corrente, anche manuale, e con eventuale origine
|
||||
documentale distinta. Possiede un identificatore stabile indipendente dal kind,
|
||||
assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono fuse
|
||||
automaticamente.
|
||||
|
||||
**Evidence kind** — La categoria semantica di una Evidence Unit, che ne determina i
|
||||
campi specifici e ne orienta l'uso. Ogni unità ha un solo kind primario; i tipi iniziali
|
||||
@@ -151,10 +209,9 @@ avanzare fino a un retry riuscito.
|
||||
nella sessione: stage semantico, purpose, generazione interrogata e identificatori delle
|
||||
Evidence restituite. Non duplica il contenuto delle Evidence.
|
||||
|
||||
**Curated Evidence** — Una o più Evidence Unit ristrutturate a partire da una Source
|
||||
Evidence e conservate nel repository del workspace come proposte per la revisione
|
||||
umana. Git conserva la versione precedente e rende visibile ogni modifica; una Curated
|
||||
Evidence non è ancora contenuto autorevole del runtime.
|
||||
**Curated Evidence** — Una o più Evidence Unit preparate da documenti o curate
|
||||
manualmente. La presenza nell'archivio curato non implica da sola che il contenuto
|
||||
sia già attivo per il workflow.
|
||||
|
||||
**Published Evidence** — Le Curated Evidence valide appartenenti alla revisione attiva
|
||||
del workspace e alla generazione Evidence pubblicata. L'approvazione umana precede
|
||||
@@ -176,9 +233,17 @@ una Evidence Unit. Il sistema ne verifica deterministicamente la presenza dopo l
|
||||
normalizzazione meccanica; il curatore resta responsabile di verificarne la sufficienza
|
||||
semantica.
|
||||
|
||||
**Evidence resolution** — L'operazione esplicita con cui un curatore ritira una
|
||||
Evidence Unit oppure la ricollega a un Source Evidence esistente. Aggiorna documento e
|
||||
manifest insieme, lascia un diff Git revisionabile e non pubblica né crea commit.
|
||||
**Evidence resolution** — La decisione esplicita con cui un curatore risolve un
|
||||
problema di una Evidence Unit, correggendola, ritirandola oppure ricollegandola a
|
||||
una fonte adeguata.
|
||||
|
||||
**Source update conflict** — Un contrasto fra una fonte aggiornata e una correzione
|
||||
manuale già approvata. La correzione resta in uso fino alla risoluzione esplicita
|
||||
del confronto da parte dell'amministratore.
|
||||
|
||||
**Evidence source refresh** — La riacquisizione delle fonti esterne richiesta
|
||||
dall'amministratore per rilevarne le modifiche. Fra due aggiornamenti il contenuto
|
||||
già acquisito resta il riferimento per preparazione e consultazione.
|
||||
|
||||
**Review item** — Un blocco di revisione descritto da codice stabile, messaggio umano e
|
||||
campo opzionale. Finché viene mantenuto nell'Evidence Unit, ne impedisce la
|
||||
@@ -230,10 +295,18 @@ conversione degli a-capo e rimozione degli spazi esterni. Gli elementi contestua
|
||||
poi deduplicati e ordinati senza conversione delle maiuscole, mentre punteggiatura e
|
||||
spazi interni della domanda non vengono riscritti.
|
||||
|
||||
**Additive BM25 upgrade** — L'estensione non distruttiva della collezione semantica di
|
||||
un workspace che conserva il vettore dense predefinito e aggiunge il solo vettore
|
||||
sparse `bm25`. Soltanto gli Evidence Fragment ricevono valori BM25; Schema e Memory
|
||||
mantengono invariati dati e ricerca dense.
|
||||
**Reference Vector Collection** — La collezione Qdrant ricostruibile di un workspace che
|
||||
contiene Schema, relazioni ed Evidence. Possiede il vettore dense predefinito e il vettore
|
||||
sparse `bm25`; soltanto gli Evidence Fragment ricevono valori BM25. Il preprocessing può
|
||||
sostituirla o eliminarla integralmente.
|
||||
|
||||
**Memory Vector Collection** — La collezione Qdrant persistente di un workspace che contiene
|
||||
`memory` e `solved_question`. Non è un output del preprocessing e non viene eliminata dal
|
||||
Preprocessing Clear.
|
||||
|
||||
**Preprocessing Clear** — L'operazione amministrativa che elimina Reference Vector Collection,
|
||||
LSH, corpus e checkpoint derivati e rende il workspace non pronto. Conserva Memory Vector
|
||||
Collection, sessioni, Catalog Metadata e database sorgente; non offre history o rollback.
|
||||
|
||||
**Formula proposal** — Una formula individuata durante una sessione e conservata come
|
||||
artefatto della sessione. Non diventa Published Evidence finché non viene importata,
|
||||
@@ -246,9 +319,10 @@ precedente o di un altro workspace.
|
||||
|
||||
## Configurazione dei modelli
|
||||
|
||||
**Workspace Descriptor** — La dichiarazione versionata dell'identità e dello scope del
|
||||
Workspace Database e delle Evidence di un workspace. Non definisce modelli, selezioni di
|
||||
modello o Database Binding specifiche di un'installazione.
|
||||
**Workspace Descriptor** — La dichiarazione versionata dell'identità del workspace e dello
|
||||
scope delle sue Evidence. Non contiene identità o configurazione del Workspace Database,
|
||||
Database Binding, fatti strutturali o metadati semantici: il Metadata Catalog associa il
|
||||
workspace al relativo database.
|
||||
|
||||
**Installation Model Catalog** — L'insieme dichiarativo, proprio di un'installazione, dei
|
||||
modelli disponibili, dei loro Model Usage e dei relativi default. È l'unica autorità per i
|
||||
@@ -302,8 +376,9 @@ client configurabile per API REST arbitrarie.
|
||||
nel catalogo autorevole. Rimane conservato per il recupero amministrativo, ma non può essere
|
||||
usato dal workflow finché non viene riassegnato a un workspace esistente.
|
||||
|
||||
**Metadata Catalog** — Il contesto amministrativo che raccoglie e cura i metadati di un
|
||||
Workspace Database. Non definisce quali elementi partecipano al workflow SQL.
|
||||
**Metadata Catalog** — L'autorità per l'associazione fra workspace e Workspace Database, la
|
||||
relativa Database Binding, i fatti strutturali osservati e i metadati semantici curati. Ogni
|
||||
uso downstream dei metadati del database deriva da questo catalogo.
|
||||
|
||||
**Database Profile** — L'insieme curato di scope, descrizioni e metadati semantici
|
||||
associato a un Workspace Database.
|
||||
@@ -365,15 +440,19 @@ logicamente.
|
||||
Logical Relationship, con origine e stato espliciti. È l'interfaccia usata dall'amministrazione e
|
||||
dalla comprensione dello schema, non un ulteriore modello persistito.
|
||||
|
||||
**Effective Relationship Snapshot** — La proiezione runtime immutabile delle relationship attive
|
||||
contenute nell'Effective Relationship Map. È derivata dal Metadata Catalog per una singola sessione
|
||||
e viene eliminata insieme alla relativa configurazione runtime.
|
||||
**Catalog Metadata Snapshot** — La proiezione immutabile e versionata della struttura catalogata,
|
||||
delle descrizioni pubblicabili e delle relazioni effettive attive di un Workspace Database che il
|
||||
core consuma. È derivata esclusivamente dal Metadata Catalog e non è un archivio autoritativo.
|
||||
|
||||
**Schema Index** — La proiezione vettoriale ricostruibile dei metadati del Workspace Database nel
|
||||
Metadata Catalog. Il preprocessing la sostituisce integralmente e non è una fonte di verità.
|
||||
|
||||
**Description** — Il testo curato e consolidato che descrive una Catalog Table o Catalog Column
|
||||
per gli usi downstream.
|
||||
per gli usi downstream. Quando presente, prevale sulla relativa Generated Description.
|
||||
|
||||
**Generated Description** — Una proposta modificabile sottoposta a revisione umana prima di
|
||||
essere consolidata come Description. Rimane distinta dal commento osservato nel database.
|
||||
**Generated Description** — Il testo modificabile prodotto dall'AI per una Catalog Table o Catalog
|
||||
Column. È pubblicabile per gli usi downstream quando manca una Description, anche senza essere
|
||||
prima consolidato, e rimane distinto dal commento osservato nel database.
|
||||
_Avoid_: generated comment, source comment
|
||||
|
||||
**Description Consolidation** — L'azione amministrativa esplicita che copia la Generated
|
||||
@@ -419,9 +498,17 @@ Schema Synchronization.
|
||||
della Database Binding. Uno scope rimane consultabile ma è stale finché non viene sincronizzato
|
||||
con la binding corrente.
|
||||
|
||||
**Metadata Content Revision** — La revisione monotona di tutto lo stato del Metadata Catalog che
|
||||
può modificare il comportamento del core. Ogni mutazione rilevante produce una nuova revisione
|
||||
nella stessa transazione che la rende durevole.
|
||||
|
||||
**Preprocessing State** — Lo stato corrente `running`, `succeeded` o `failed` del preprocessing di
|
||||
un workspace, insieme all'identità dei suoi input. Il core può usare il workspace soltanto quando
|
||||
lo stato è `succeeded` e gli input coincidono ancora.
|
||||
|
||||
**Catalog Metadata** — I campi mutabili che descrivono database, tabelle, colonne e relazioni,
|
||||
distinti dai fatti strutturali governati dalla sincronizzazione. Possono essere popolati dall'AI,
|
||||
da un'importazione o da una modifica amministrativa senza cambiare il database esterno.
|
||||
o da una modifica amministrativa senza cambiare il database esterno.
|
||||
|
||||
**Model Completion Helper** — Il processo Python interno ed effimero che esegue una singola
|
||||
richiesta LiteLLM per conto del backend. Non è un servizio HTTP, non possiede il lifecycle della
|
||||
@@ -484,3 +571,73 @@ _Avoid_: Sensitive Data Suggestion Event
|
||||
**Introspection Capability** — Una categoria di struttura fisica che una Database Binding
|
||||
può osservare, come tabelle, colonne, relazioni, indici o enum. Una capability non disponibile
|
||||
è distinta da una capability osservata che non ha restituito elementi.
|
||||
|
||||
## Amministrazione e integrazione
|
||||
|
||||
**Workspace Readiness** — La preparazione di uno specifico Workspace per l'uso nel
|
||||
workflow, comprensiva della disponibilità degli artefatti derivati dai suoi metadati
|
||||
Database e dalle sue Evidence. Il preprocessing appartiene a questa preparazione;
|
||||
la configurazione e la sincronizzazione del catalogo restano responsabilità Database.
|
||||
|
||||
**Administration Surface** — Una superficie amministrativa autonoma per configurare o curare una
|
||||
parte dell'installazione. Workspace, Evidence, Memory, Database e Pi sono superfici peer e non
|
||||
dipendono dall'esistenza di una sessione attiva.
|
||||
|
||||
**Administration Page** — La rappresentazione a pagina intera di una Administration Surface, con
|
||||
una gerarchia condivisa per identità, stato, azioni e contenuto. Un form amministrativo appartiene
|
||||
alla pagina e non a una popup come contenitore principale.
|
||||
_Avoid_: management popup, settings modal
|
||||
|
||||
**Administration Route** — L'identità navigabile di una Administration Surface nel browser. Deve
|
||||
essere ripristinabile con refresh e cronologia e non contiene valori transitori o segreti dei form.
|
||||
|
||||
**Embedded Thoth Shell** — L'esperienza Thoth ospitata dentro il documento e il contesto visuale di
|
||||
un portale host. Conserva la propria gerarchia funzionale, ma deve rispettare la geometria,
|
||||
l'autenticazione e le regole responsive del portale host.
|
||||
|
||||
**Full Thoth Shell** — L'esperienza Thoth autonoma che possiede il proprio header e il proprio
|
||||
layout di pagina. Non replica la navigazione amministrativa del portale host e non dipende dal suo
|
||||
template visuale.
|
||||
|
||||
**Shell mode** — La scelta di installazione fra `embedded` e `full`. Determina chi possiede il
|
||||
chrome globale, i comandi di identità e le integrazioni visuali, ma non cambia il workflow o la
|
||||
persistenza delle sessioni.
|
||||
|
||||
**Fullscreen state** — Lo stato temporaneo in cui il documento applicativo occupa il fullscreen
|
||||
del browser. È distinto da `Shell mode`: una Full Thoth Shell può essere aperta senza fullscreen;
|
||||
il passaggio è attivato da un comando esplicito e può essere annullato con la stessa azione o con
|
||||
il comando nativo del browser.
|
||||
|
||||
**Portal Shell Adapter** — Il confine sostituibile che traduce lo stato e i comandi del chrome di
|
||||
un portale host nel modello semantico usato da Thoth. L'adapter non possiede autorizzazione,
|
||||
sessioni di workflow o contenuti del modello.
|
||||
|
||||
**Host Shell State** — Il minimo stato visuale fornito dal portale host: locale UI, tema e stato
|
||||
fullscreen. In una Embedded Thoth Shell è la fonte autorevole per queste preferenze;
|
||||
non include identità, token o stato di autenticazione, che restano responsabilità dell'accesso.
|
||||
|
||||
**UI locale** — La lingua delle label, dei messaggi, dei tooltip, degli stati e delle istruzioni
|
||||
non generate dal modello nell'interfaccia Thoth. È distinta dalla lingua dei contenuti di un
|
||||
workspace.
|
||||
|
||||
**Interaction language** — La lingua in cui il modello presenta domande, spiegazioni e proposte
|
||||
al revisore durante una sessione. Viene fissata alla creazione della sessione e rimane invariata
|
||||
durante una ripresa, anche se la UI locale corrente cambia.
|
||||
|
||||
**Administrative Page Family** — L'insieme delle cinque Administration Page che condividono shell,
|
||||
navigazione, tipografia e regole responsive, pur mantenendo contenuti e operazioni specifici:
|
||||
Workspace, Evidence, Memory, Database e Pi.
|
||||
|
||||
## Installazione
|
||||
|
||||
**Manual standalone installation** — Una copia di ThothII predisposta per l'uso autonomo da una
|
||||
persona che possiede il computer, con una Full Thoth Shell e servizi applicativi locali. La
|
||||
procedura non implica che DWH o provider LLM siano locali o disponibili offline.
|
||||
|
||||
**Installation bootstrap** — L'insieme delle attività iniziali che rende disponibile una
|
||||
installazione manuale: verifica dell'host, generazione della configurazione, predisposizione
|
||||
delle credenziali protette e avvio dei servizi. Non è un installer dell'applicazione.
|
||||
|
||||
**Platform acceptance** — La verifica che una Manual standalone installation possa essere
|
||||
predisposta e avviata su una specifica combinazione di sistema operativo, architettura e runtime,
|
||||
distinta dalla verifica funzionale del collegamento a DWH e provider LLM.
|
||||
|
||||
@@ -11,50 +11,50 @@ colors:
|
||||
warm-graphite: "oklch(26.78% 0.0097 355.6)"
|
||||
muted-graphite: "oklch(51.33% 0.0088 345.6)"
|
||||
quiet-border: "oklch(90.93% 0.0035 354.7)"
|
||||
success-mint: "oklch(75.77% 0.1581 165)"
|
||||
success-mint: "oklch(46% 0.095 160)"
|
||||
navigation-active: "oklch(92.5% 0.052 23.2)"
|
||||
navigation-active-hover: "oklch(89.5% 0.071 23.2)"
|
||||
navigation-active-foreground: "oklch(36.5% 0.11 23.2)"
|
||||
navigation-active-border: "oklch(60% 0.135 23.2)"
|
||||
warning-amber: "oklch(85.23% 0.1386 78.9)"
|
||||
information-blue: "oklch(70.35% 0.1128 221.3)"
|
||||
warning-amber: "oklch(48% 0.09 70)"
|
||||
information-neutral: "oklch(51.33% 0.0088 345.6)"
|
||||
typography:
|
||||
display:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "3rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.5rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.03
|
||||
letterSpacing: "-0.025em"
|
||||
headline:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "1.875rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.5rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.15
|
||||
letterSpacing: "-0.015em"
|
||||
title:
|
||||
fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
|
||||
fontSize: "1.2rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.25rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "-0.01em"
|
||||
body:
|
||||
fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "0.9375rem"
|
||||
fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "1rem"
|
||||
fontWeight: 400
|
||||
lineHeight: 1.65
|
||||
letterSpacing: "normal"
|
||||
control:
|
||||
fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "0.875rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "0.005em"
|
||||
label:
|
||||
fontFamily: "ui-monospace, SF Mono, Cascadia Code, Menlo, Consolas, monospace"
|
||||
fontSize: "0.6875rem"
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "0.75rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "0.06em"
|
||||
letterSpacing: "normal"
|
||||
rounded:
|
||||
xs: "4px"
|
||||
sm: "6px"
|
||||
@@ -113,6 +113,16 @@ components:
|
||||
|
||||
# Design System: ThothII
|
||||
|
||||
## Visual review branch, September 2026
|
||||
|
||||
The revision on `codex/ui-visual-review` is approved for implementation and Docker visual review,
|
||||
not yet for adoption on `main`. The previous look remains recoverable from the base commit and
|
||||
the preserved Docker image. Historical prototypes must remain untouched.
|
||||
|
||||
This revision follows Impeccable's product register: one locally bundled Manrope family for the
|
||||
whole UI, five fixed size roles, red as the sole brand accent and additional color only for meaningful
|
||||
state. The primary scene remains an analyst reading data and SQL in a well-lit office.
|
||||
|
||||
## Overview
|
||||
|
||||
**Creative North Star: "The Clinical Workbench"**
|
||||
@@ -134,7 +144,7 @@ disciplined and tactile, never playful, sluggish, or visually unstable.
|
||||
**Key Characteristics:**
|
||||
|
||||
- Warm, restrained surfaces with one scarce red accent.
|
||||
- Editorial headings paired with highly legible operational body text.
|
||||
- One sans-serif family, with hierarchy expressed through size, weight and spacing.
|
||||
- Dense information organized through hierarchy, rhythm, and progressive disclosure.
|
||||
- Persisted artifacts and reviewer decisions presented as the visual source of truth.
|
||||
- Fast state feedback with reduced-motion parity.
|
||||
@@ -150,6 +160,15 @@ users can scan structure without adding nested containers.
|
||||
|
||||
## Colors
|
||||
|
||||
The full-mode application header matches Omics Portal's `--gsd-red-primary`
|
||||
(`#CB333B`) in both themes. Its complete wordmark, including `II`, and controls
|
||||
use a near-white foreground. This header is absent in embedded mode. The sidebar
|
||||
and welcome wordmarks retain their red suffix. Context editing places workspace,
|
||||
model and Done in one desktop row, stacking on narrow containers. Session-scope
|
||||
tabs retain their selected fill and accessible keyboard state with a uniform one-pixel
|
||||
border on every side, gray when inactive and red when active. Their padding is 11px
|
||||
horizontal and 3px vertical, with a 38px minimum height and wrapping labels.
|
||||
|
||||
The palette combines warm porcelain surfaces, warm graphite text, and an instrument red used only
|
||||
for action, focus, and important state. OKLCH values in the frontmatter are normative because the
|
||||
frontend uses OKLCH tokens directly.
|
||||
@@ -178,7 +197,8 @@ frontend uses OKLCH tokens directly.
|
||||
foreground. It shares Instrument Red's hue but uses a lighter, lower-chroma fill, so location is
|
||||
visible without carrying the full weight of a primary action.
|
||||
- **Warning Amber** (`warning-amber`): waiting, attention, and in-progress states.
|
||||
- **Information Blue** (`information-blue`): informational state when red would imply action.
|
||||
- **Information**: neutral text and indicators for dates, protocols and ordinary status. The legacy
|
||||
`--info` token resolves to muted foreground, not an additional blue accent.
|
||||
|
||||
The dark theme keeps the same semantic mapping with neutral near-black surfaces and a slightly
|
||||
lighter red accent. Do not introduce a second visual identity for dark mode.
|
||||
@@ -191,30 +211,33 @@ for their named states. Color is never the only state indicator.
|
||||
|
||||
## Typography
|
||||
|
||||
**Display Font:** Fraunces, with Source Serif Pro, Georgia, and Times New Roman fallbacks
|
||||
**Body Font:** Manrope, with native system sans-serif fallbacks
|
||||
**Label/Mono Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks
|
||||
**UI Font:** locally bundled Manrope Variable, with Manrope and native sans-serif fallbacks.
|
||||
**Technical Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks.
|
||||
|
||||
**Character:** Fraunces gives persisted artifacts and key headings editorial authority. Manrope
|
||||
keeps dense controls and prose calm and readable. The mono register separates machine identity,
|
||||
metadata, SQL, identifiers, and micro-labels from natural-language content.
|
||||
Manrope covers headings, labels, controls, navigation and document reading. Monospace is reserved
|
||||
for SQL, code, paths and machine identifiers, never for ordinary UI labels or status headings.
|
||||
|
||||
### Hierarchy
|
||||
|
||||
- **Display** (600, `3rem`, `1.03`): authentication and exceptional page-level statements only.
|
||||
- **Headline** (600, `1.875rem`, `1.15`): major page or artifact titles.
|
||||
- **Title** (600, `1.2rem`, `1.25`): panel and document section hierarchy.
|
||||
- **Body** (400, `0.9375rem`, `1.65`): operational prose, with a target line length of 65 to 75
|
||||
- **Headline** (600, `1.5rem`, `1.3`): page or artifact titles, `--text-page`.
|
||||
- **Title** (600, `1.25rem`, `1.4`): section hierarchy, `--text-section`.
|
||||
- **Body** (400, `1rem`, `1.6`): operational prose, `--text-body`, with a target line length of 65 to 75
|
||||
characters where the surface controls width.
|
||||
- **Control** (600, `0.875rem`, `1.25`): buttons, inputs, tabs, and compact actions.
|
||||
- **Label** (600, `0.6875rem`, `0.06em` tracking): uppercase micro-labels, state metadata, and panel
|
||||
headers. Labels use the mono family.
|
||||
- **Control** (400–600, `0.875rem`, `1.5`): buttons, inputs, tables, tabs and compact subheadings,
|
||||
`--text-control`.
|
||||
- **Metadata** (400–600, `0.75rem`, `1.5`): secondary status, counts and timestamps, `--text-meta`.
|
||||
Labels use sentence case and normal tracking. Ordinary operational text never falls below 12px.
|
||||
|
||||
Typography uses fixed sizes. Responsive changes happen at structural breakpoints, not through fluid
|
||||
type scaling. Numeric data and identifiers use tabular numerals where comparison matters.
|
||||
|
||||
**The Three Registers Rule.** Serif means authority, sans means interaction and reading, mono means
|
||||
machine identity. Do not exchange these roles for novelty.
|
||||
**Application wordmark:** ThothII is a brand mark, not a page title: use Manrope semibold at
|
||||
48px (`3rem`) in the Core welcome area and 32px (`2rem`) in the session sidebar, with the
|
||||
`II` suffix in brand red. Preserve these sizes across responsive layouts.
|
||||
|
||||
**The One Family Rule.** The UI and document readers use sans-serif throughout. The legacy
|
||||
`--font-heading` alias resolves to `--font-sans`. Preserve technical monospace without turning it
|
||||
into a second decorative hierarchy. Do not shrink text to solve layout constraints.
|
||||
|
||||
**The Read Once Rule.** A heading, label, and body must be distinguishable on first glance through
|
||||
size and weight. Do not repeat headings in explanatory copy.
|
||||
@@ -279,27 +302,70 @@ default, hover, focus, active, disabled, loading, and error behavior where those
|
||||
- **Focus:** three-pixel Instrument Red ring with a clear border shift.
|
||||
- **Error / Disabled:** errors combine destructive color with explanatory text; disabled controls
|
||||
retain readable contrast and use 50 percent opacity.
|
||||
- **Metadata catalog model:** Database Management keeps one compact, installation-level
|
||||
metadata-generation LLM selector in the application header. The selection persists across
|
||||
database, table, column, and relationship views; when no usable profile is configured, the
|
||||
disabled control explains: “No metadata-generation LLM model is configured for this installation.”
|
||||
- **Global context:** the collapsible top shelf is the sole workspace/model selector for Core and
|
||||
Admin. Preserve independent remembered choices, installation defaults, operation locks and unsaved
|
||||
edit guards. Never introduce a separate metadata-generation default or selector.
|
||||
|
||||
### Navigation
|
||||
|
||||
- **Workspace readiness:** the Workspace navigation button carries an 8px dot to
|
||||
the right of its label. Green means a selected workspace with confirmed ready
|
||||
preprocessing and no query error; all other states are red. The button's
|
||||
tooltip and accessible description retain the translated exact state. Do not
|
||||
add a separate readiness text row or change the backend readiness gate.
|
||||
- **Session groups:** one accessible single-open accordion contains Active sessions
|
||||
and Archive, both initially closed. Below the scope tabs, show only their
|
||||
adjacent section headers, without a redundant Sessions heading. Selection and
|
||||
bulk-delete controls belong inside each panel and only appear for nonempty
|
||||
lists. Select all affects that list only, preserves the other list's selection,
|
||||
and exposes a mixed state for partial selection. Preserve the existing archived
|
||||
flag as the grouping rule, independent of whether a Pi process is running.
|
||||
Opening a section closes the other; either can be collapsed, including both.
|
||||
Empty lists show only the translated "No sessions yet." message.
|
||||
The open section uses the rail's remaining height; its list scrolls internally
|
||||
with a cap of `min(18rem, 35dvh)`, while its trigger remains outside that scroll
|
||||
area. The mobile navigation dialog supplies a bounded viewport-height container.
|
||||
Keyboard users can focus and scroll each labelled panel.
|
||||
- **Session entry:** one Session button returns to the current unfinished session,
|
||||
including provisional creation, without resetting or reconnecting it. Otherwise
|
||||
it prepares a new question using the normal readiness and unsaved-work guards.
|
||||
- **Style:** compact session rows use `8px` corners and restrained vertical padding.
|
||||
- **Default / Hover / Active:** porcelain at rest, Sunken Surface on hover, and a muted Navigation
|
||||
Active red with a defined border when current. Exactly one top-level navigation control is current.
|
||||
- **Administrative controls:** the admin-only Administration accordion groups Database management,
|
||||
a structural divider, Workspace management, and Pi management in that order. Its trigger exposes
|
||||
- **Administrative controls:** the admin-only Administration accordion groups Database,
|
||||
Memory, Evidence, a structural divider, Workspace, and Pi configuration in that order. Its trigger exposes
|
||||
expanded state and starts collapsed by default, while non-admin users do not receive the accordion
|
||||
or its navigation actions.
|
||||
- **Responsive:** collapse navigation structurally at the application breakpoint. Do not shrink
|
||||
labels into illegibility.
|
||||
labels into illegibility. Below 768px, Memory and Evidence management use the full content
|
||||
width; a Navigation button opens the shared accessible dialog. Selecting another archive
|
||||
page or pressing Escape closes it. Desktop retains the right session sidebar and its My sessions /
|
||||
All sessions tabs. Core retains question/answer, eight phases, reviewer gates and the left log.
|
||||
In embedded mode the portal owns the red header and left sidebar; ThothII must not duplicate them. Size to the
|
||||
actual application container. Narrow session document panels may use the available width.
|
||||
|
||||
### Session review and confirmations
|
||||
|
||||
Session dialogs use the visible application area, including the portal's header
|
||||
and side rail. Artifact and schema-column review can grow to 80rem wide and the
|
||||
available height; short confirmations use up to 40rem and at least 18rem when
|
||||
space permits. Keep a 24px outer margin on desktop and 8px on small or short
|
||||
screens. Long review content scrolls internally; on very short screens the
|
||||
whole dialog can also scroll so every action remains reachable.
|
||||
|
||||
Session forms and review gates repeat their existing primary confirmation above
|
||||
and below the content, sharing selection, validation, pending state and response
|
||||
handlers. Alternate-response inputs follow the same rule. Reserved navigation
|
||||
controls remain below the review. Stop/delete initially focus Cancel; rename
|
||||
initially focuses the name field. Administration dialogs and forms retain their
|
||||
existing layout and actions.
|
||||
|
||||
### Tabs
|
||||
|
||||
- **Shape:** compact label tabs sit on a shared baseline with rounded top corners and a two-pixel
|
||||
lower edge. Inactive labels retain a complete Quiet Border and Porcelain Card surface, so every
|
||||
lower edge, except session-scope tabs which use a uniform one-pixel border, rounded
|
||||
corners and a 4px gap without a shared border or negative bottom margin.
|
||||
Inactive labels retain a Quiet Border and Porcelain Card surface, so every
|
||||
label reads as a tab before interaction; hover feedback reinforces clickability.
|
||||
- **Current:** the selected tab uses the muted Navigation Active red for its fill, text, and defined border.
|
||||
It must expose `aria-selected`, participate in a labelled `tablist`/`tabpanel`, and be the only
|
||||
@@ -319,9 +385,42 @@ default, hover, focus, active, disabled, loading, and error behavior where those
|
||||
|
||||
### Curated Evidence Documents
|
||||
|
||||
Memory and Evidence share the `thot-knowledge-reader` reading contract. Use locally
|
||||
bundled Manrope with normal tracking for prose and labels, and these fixed roles:
|
||||
|
||||
- Card title: 24px, weight 600, line-height 1.3 (`thot-knowledge-title`).
|
||||
- Field/section heading, including Scope and Provenance: 20px, weight 600,
|
||||
line-height 1.4, 8px clearance below (`thot-knowledge-heading`).
|
||||
- All narrative text, including scope, lists and provenance: 16px, weight 400,
|
||||
line-height 1.65. Do not apply compact UI text sizes to these fields.
|
||||
- Authored Markdown subheadings inside a field: 16px, weight 600, line-height 1.5,
|
||||
24px above/8px below. They remain subordinate to the enclosing field heading;
|
||||
their semantic heading levels and original content are preserved.
|
||||
- Technical metadata labels/values: 14px/1.5, with weight 600 for labels.
|
||||
Only code, paths and machine identifiers use the technical monospace family at
|
||||
14px/1.65, identical for inline and fenced code (never compound `em` shrinkage).
|
||||
|
||||
Separate reading sections by 24px; keep the first Markdown block flush with its
|
||||
field heading's 8px bottom gap. The same typography applies in light/dark and at
|
||||
all responsive widths. Controls and archive indexes retain their compact UI roles.
|
||||
|
||||
Memory and Evidence detail readers use the entire available content width, without
|
||||
the ordinary 72–75ch prose cap. This is the owner's explicit reading-layout choice.
|
||||
Long unstructured paragraphs are split for display at existing sentence/semicolon
|
||||
boundaries outside inline code and links; authored Markdown structure and stored
|
||||
content are unchanged. Paragraph spacing is 1.25em. Scope and provenance share the
|
||||
available width; provenance excerpts render Markdown rather than literal markers.
|
||||
Copy actions use the two-overlapping-sheets icon, an accessible name/tooltip and
|
||||
live success/failure feedback instead of a visible Copy label.
|
||||
|
||||
Memory has four explicitly FAKE formatting examples, one per family, in a separate
|
||||
expandable section. They reuse the real detail reader but never enter persistence,
|
||||
indexing, link search or model recall, and expose no edit/delete/save actions.
|
||||
|
||||
Curated evidence follows a fixed reading order: title, compact type and purpose summary, scope,
|
||||
typed content, supporting excerpts, review items, then collapsed technical provenance. Machine
|
||||
metadata stays in invisible comments so GitHub Preview shows only the reviewable document.
|
||||
typed content, supporting excerpts, review items, then technical provenance. Curated v4 files
|
||||
use short, visible YAML frontmatter for identity and classification. The Markdown title and
|
||||
body are authoritative; hidden payload comments are a legacy format converted on consolidation.
|
||||
|
||||
`applies_to` is rendered as “Ambito di applicazione” with separate bullet lists for concepts,
|
||||
tables, and columns. Enum values also use lists. Tables are forbidden for metadata, scope, or any
|
||||
@@ -329,7 +428,9 @@ one-dimensional collection; reserve tables for genuinely two-dimensional dataset
|
||||
identifiers use inline code. SQL uses fenced code. Supporting excerpts use blockquotes.
|
||||
|
||||
**The Review Surface Rule.** The visible Markdown must be readable without understanding the
|
||||
machine contract. Technical metadata belongs in progressive disclosure, not above the title.
|
||||
machine contract. In Administration, explain current and original provenance separately and
|
||||
keep file-editing templates and Git instructions in progressive disclosure. Show actual host
|
||||
paths with copy controls, never browser file links to container-only locations.
|
||||
|
||||
## Do's and Don'ts
|
||||
|
||||
@@ -341,7 +442,8 @@ machine contract. Technical metadata belongs in progressive disclosure, not abov
|
||||
- **Do** preserve information density with headings, rhythm, and progressive disclosure.
|
||||
- **Do** keep keyboard focus explicit and pair color with text, shape, icon, or position.
|
||||
- **Do** respect `prefers-reduced-motion` while preserving immediate non-kinetic feedback.
|
||||
- **Do** use English for interface chrome and the workspace language for persisted document content.
|
||||
- **Do** use the selected interface language (English by default) for chrome and preserve the
|
||||
workspace language for persisted domain content. Session interaction language remains pinned.
|
||||
- **Do** render curated metadata and scope as Markdown prose or lists, never as a frontmatter table.
|
||||
- **Do** break long curated rules into paragraphs, labelled subsections, and lists at existing
|
||||
punctuation boundaries while preserving the exact canonical text for machines.
|
||||
|
||||
+84
-278
@@ -1,295 +1,101 @@
|
||||
# ThothII — Project State
|
||||
# Project state
|
||||
|
||||
Last updated: 2026-09-04.
|
||||
Updated: 2026-09-27. This is a current snapshot, not a release diary. Stable commands
|
||||
and invariants are in [AGENTS.md](AGENTS.md); prior snapshots remain in Git.
|
||||
|
||||
This file is the short operational snapshot. Stable commands and the architecture mental model
|
||||
live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`,
|
||||
`docs/contracts/`, `docs/adr/`, and `docs/evidence.md`. Superseded plans and reports are
|
||||
available from Git history rather than duplicated in the working tree.
|
||||
## Current contracts
|
||||
|
||||
The guarded server migration from a legacy checkout to the schema-v2 installation, Gitea source,
|
||||
Authentik, internal catalog/embedding services, and the PSD workspace repository is documented in
|
||||
`docs/operations/server-upgrade-gitea-workspace-v2.md`. Treat its operator gates and rollback
|
||||
requirements as mandatory; do not replace the running server stack in place.
|
||||
- React supports full/embedded rendering independently of local/OIDC/upstream auth,
|
||||
with EN/IT UI and immutable session interaction language. See
|
||||
[application shell](docs/architecture/application-shell.md) and
|
||||
[localization](docs/operations/shell-and-localization.md).
|
||||
- PostgreSQL Metadata Catalog owns database identity, binding, schema, descriptions,
|
||||
sensitivity and relationships for all core consumers. Workspace schema v4 contains
|
||||
identity and optional Evidence only. Installation schema v2 is the authored model
|
||||
catalog source. See [overview](docs/architecture/overview.md) and
|
||||
[model configuration](docs/general/pi-configuration.md).
|
||||
- The harness owns workflow persistence; chat is not the durable session record.
|
||||
Memory uses PostgreSQL authority and derived Qdrant dense/BM25 search. Editable
|
||||
Evidence has local archive authority and manual consolidation. File save, search
|
||||
activation and Git publication have distinct outcomes. See
|
||||
[Memory](docs/gestione-memory.md), [Evidence](docs/contracts/curated-evidence-v4.md)
|
||||
and [consolidated release evidence](docs/reports/knowledge-archives-release.md).
|
||||
- Reference preprocessing must not clear Memory. Use the installation-scoped
|
||||
`tht --installation /absolute/path/thothii-installation.yaml workspace preprocess run`
|
||||
and its [contract](docs/contracts/workspace-preprocessing-cli.md).
|
||||
- DWH sessions are read-only. SSH tunnels support database-management diagnostics
|
||||
and metadata synchronization, not NL→SQL session creation; use direct or REST
|
||||
transport for sessions.
|
||||
|
||||
## Current product shape
|
||||
## Installation and workspace boundaries
|
||||
|
||||
ThothII is a human-in-the-loop datamart builder with three independently built layers:
|
||||
Fresh standalone installations follow the manual terminal procedures in
|
||||
[Italian](docs/install/standalone-manual-it.md) or
|
||||
[English](docs/install/standalone-manual-en.md), without an installer or launcher.
|
||||
The [Compose reference](docs/operations/compose-reference.md) is for maintainers,
|
||||
not another quick start. Catalog and Memory migrations are explicit.
|
||||
|
||||
```text
|
||||
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
|
||||
```
|
||||
Descriptors, authentication, provider credentials, certificates and runtime bindings
|
||||
stay in protected installation-local paths. Do not copy secrets into examples or
|
||||
workspace Git. Legacy runtime snapshots may use absolute paths and protected
|
||||
`harness/.env`; do not silently relocate them.
|
||||
|
||||
The harness owns the deterministic eight-phase NL→SQL workflow and all session persistence.
|
||||
The backend remains a process/RPC/SSE bridge for sessions and now also owns an isolated PostgreSQL
|
||||
metadata catalog for administrative database configuration. The frontend renders the review gates
|
||||
and keeps the live transcript in memory. See
|
||||
`docs/architecture/components.md` for the detailed component and data-flow map.
|
||||
PSD authoring is separate at `/Users/mp/projects/tht-workspace-psd`. Its GitHub
|
||||
repository was copied to private Gitea
|
||||
[workspace_psd](https://git.tylconsulting.it/mptyl/workspace_psd), preserving both
|
||||
branches. It is a copy, not automatic synchronization. Running installations were
|
||||
not repointed to a different workspace remote.
|
||||
|
||||
## Evidence restructuring — accepted
|
||||
## Recorded deployments and server authority
|
||||
|
||||
The evidence restructuring and PSD migration completed real acceptance on 2026-08-25.
|
||||
Use the ordered [server handoff](docs/operations/server-codex-handoff.md) for
|
||||
coordinated ThothII/Omics upgrades. Omics integration uses GitHub
|
||||
`Dallavilla-Tiziano/omics_portal`, with no Gitea relay prerequisite. Omics uses
|
||||
embedded/upstream identity, not another ThothII OIDC login. Read
|
||||
[upstream authentication](docs/install/authentication-upstream.md) before changes.
|
||||
|
||||
- The curated PSD revision contains 35 approved Evidence units and 60 review items.
|
||||
- The PSD authoring repository publishes all 35 units using Curated unit schema v3. Its table-free
|
||||
presentation uses hidden canonical metadata, wrapping Markdown scope lists, list-based enum
|
||||
values, and collapsed technical provenance. Long domain rules now have a deterministic
|
||||
human-readable presentation while retaining their exact canonical text for vector ingestion.
|
||||
`tht evidence migrate <workspace-root>` performs the deterministic v1/v2 upgrade and older-v3
|
||||
presentation rewrite without model calls. The structured-rule PSD rewrite is currently local and
|
||||
pending commit/publication.
|
||||
- The accepted snapshot is
|
||||
`psd-clinical-675990d90eae51da6f2bd51b1ae2609f245772ef-snapshot`.
|
||||
- The active generation is `gen:f968b3bd7a553dbfef3cf47093698f2bc7f95f11`.
|
||||
- Retrieval acceptance reached 20/20 Hit@10.
|
||||
- A real session, `20301df7-cad7-403d-a4c1-9f35c9d07b66`, completed F1–F8 with five
|
||||
receipts, three CTEs, and a final result of 78 patients.
|
||||
- The durable acceptance record is
|
||||
`docs/testing/evidence/evidence-restructuring-psd-acceptance-2026-08-25.md`.
|
||||
Last recorded application deliveries (not a fresh runtime attestation):
|
||||
|
||||
The canonical authoring, validation, publication, materialization, and preprocessing flow is
|
||||
documented in `docs/evidence.md`. The governing contracts are
|
||||
`docs/contracts/workspace-evidence-v3.md` and
|
||||
`docs/contracts/workspace-preprocessing-cli.md`.
|
||||
- [Coordinated ThothII/Omics release](docs/reports/2026-09-26-server-release-execution.md):
|
||||
core/frontend `497ab840-preflight`, Omics proxy fix `928f7e9f` with existing web
|
||||
image retained. Automated acceptance passed; on September 27 the operator confirmed
|
||||
browser access, UI controls and session start/stop/resume. Functional browser
|
||||
acceptance passed; remaining extended checks are handed off in the
|
||||
[server acceptance follow-up](docs/reports/2026-09-27-server-acceptance-handoff.md).
|
||||
- [Session dialogs](docs/reports/2026-09-14-session-dialogs-release.md):
|
||||
`b1723c34-session-dialogs-20260914`, frontend-only.
|
||||
- [Session layout/Memory fix](docs/reports/2026-09-14-session-layout-memory-fix.md):
|
||||
`49333a2d-session-memory-fix`, core/frontend.
|
||||
|
||||
## Workspace preprocessing and configuration
|
||||
Keep those reports and rollback instructions while operator gates remain open.
|
||||
A later deployment does not prove every earlier acceptance item passed.
|
||||
|
||||
The native host CLI `tht` is the operator surface. Workspace preprocessing runs through:
|
||||
## Remaining acceptance and design gates
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
|
||||
```
|
||||
- Fresh-machine Mac, Windows/WSL2 and Linux installation acceptance, including real
|
||||
DWH/model endpoints, remains a separate operator exercise.
|
||||
- Real IdP/portal login, logout, embedded interaction and PSD semantic acceptance
|
||||
follow the [manual matrix](docs/testing/authentication-manual-acceptance.md) and
|
||||
delivery reports; synthetic tests do not close them.
|
||||
- [Security hardening](docs/plans/2026-09-08-security-hardening-prd.md) is a draft.
|
||||
Revalidate SEC01–12 and obtain design approval before implementation or real
|
||||
server/IdP/DWH mutation.
|
||||
- Optional NER remains opt-in; labeled Italian quality, benchmark and licensing
|
||||
acceptance are not implied by document cleanup.
|
||||
- Semantic aliases, value descriptions, synonyms/concepts, dialect and multi-schema
|
||||
extensions remain explicit design work. Current sensitivity delivery follows the
|
||||
Catalog contract; additional policies require their own acceptance.
|
||||
- Legacy database UI fallback (`?db-ui=legacy`, dev/staging) and prototype removal
|
||||
remain subject to owner acceptance.
|
||||
|
||||
These commands use the profile-gated `workspace-maintenance` service. The former standalone
|
||||
preprocessing Compose fixtures are retired.
|
||||
## Documentation maintenance
|
||||
|
||||
Workspace descriptors use schema v4 and contain only database, Evidence, diagnostics, and binding
|
||||
concerns; model, provider, embedding, and vector-store configuration is installation-owned. For
|
||||
PSD, workspace content and runtime roots point to the
|
||||
separate uncommitted repository `/Users/mp/projects/tht-workspace-psd`. Secrets remain outside
|
||||
Git and are supplied only through installation-local protected files.
|
||||
MkDocs publishes only 20 product/operator pages and five approved assets.
|
||||
Architecture, contracts, ADRs, plans, research, tests and release evidence are
|
||||
excluded from HTML and search. The repository itself is public: editorial exclusion
|
||||
is not confidentiality.
|
||||
|
||||
## Installation Model Catalog
|
||||
|
||||
`thothii-installation.yaml` schema version 2 is the only operator-authored source for session,
|
||||
metadata-generation, and embedding models. The host `tht` lifecycle validates `modelCatalog` and
|
||||
regenerates the backend catalog, Pi `models.json`/`settings.json`, and Compose override under the
|
||||
installation-local `generated/` directory. Those projections are replaceable runtime adapters:
|
||||
they are not edited, backed up, or treated as configuration.
|
||||
|
||||
Session and metadata defaults use canonical `provider/model` IDs. Provider authentication declares
|
||||
one explicit mode (`secret_env`, `pi_auth`, or `none`); `secret_env` names a protected bundle key.
|
||||
The backend settings store now owns only the selected workspace and thinking level. Existing v1
|
||||
installations use the explicit catalog migration command; schema-v3 workspace descriptors are
|
||||
converted deterministically in their curator-owned repository before commit. Strict runtime loading
|
||||
does not silently infer or merge legacy sources. ADR 0013 and
|
||||
`docs/plans/2026-09-02-installation-model-catalog.md` record the decision and implementation.
|
||||
|
||||
## Database management
|
||||
|
||||
The database, table, and authoritative physical-schema catalog slices are implemented. Database
|
||||
Management now opens the Fleet Ledger presentation by default inside `AppShell`, lists every YAML
|
||||
workspace, creates at most one PostgreSQL database configuration per workspace, edits direct
|
||||
PostgreSQL, REST API, or SSH-tunnel installation bindings, replaces write-only encrypted secrets,
|
||||
and tests supported connector bindings. The surface keeps one responsive AG Grid visible at a time:
|
||||
databases lead to tables, tables lead to columns, and relationships are a sibling database view.
|
||||
Parent navigation remains explicit through the breadcrumb and emphasized back control.
|
||||
|
||||
Selection-scoped operations use one action selector plus an explicit **Run** control; ineligible
|
||||
actions remain visible with their disabled reason, while row-scoped actions stay in the pinned final
|
||||
column. The KPI strip reads installation-wide or selected-database aggregates from
|
||||
`GET /catalog/metrics`. Database configuration, metadata editors, synchronization history,
|
||||
description history, and sensitive-field review/history use the production APIs in right-side
|
||||
drawers rather than prototype fixtures; closing a history drawer does not stop its background run.
|
||||
|
||||
Sensitive-field review is now driven by the versioned local `sensitivity-v4` policy, not by a
|
||||
catalog model. The backend reads selected source tables through read-only, database-specific
|
||||
adapters and makes every `sensitive | non_sensitive` draft decision in the TypeScript
|
||||
`SensitivityClassifier`. A single validated match protects the column. Tables up to 1,000 rows are
|
||||
fully scanned; larger tables use breadth-first 300, 1,000, and text-only 3,000-value targets, with a
|
||||
five-second limit per source query and no global request deadline. Source failures fail the run
|
||||
instead of yielding `unknown`; coverage remains visible separately from the proposal. Draft
|
||||
assessments remain transient until an administrator explicitly saves them. Optional GLiNER2
|
||||
evidence is CPU-only, offline, opt-in, and never replaces the deterministic decision point; see
|
||||
`docs/operations/sensitivity-analysis.md`. The earlier v1 PSD shadow comparison kept NER disabled by
|
||||
default; see `docs/reports/2026-09-02-psd-sensitivity-shadow.md`. The v2 comparison completed all
|
||||
2,275 columns: CPU NER added 18 sensitive proposals and increased warm runtime from 50.1 to 61.3
|
||||
seconds; see `docs/reports/2026-09-03-psd-progressive-sensitivity-shadow.md`.
|
||||
Version 4 excludes declared `bigint` primary-key columns and conventionally named `pk bigint`
|
||||
columns before source inspection, reporting both as non-informative structural identifiers while
|
||||
distinguishing declared constraints from inferred roles.
|
||||
|
||||
Physical membership, source
|
||||
comments, column types/default/nullability/PK positions, and constraint-level ordered FK pairs are
|
||||
projections of the external schema. They cannot be created, renamed, or structurally edited by
|
||||
hand, but administrators can explicitly clear catalog tables, columns, or relationships without
|
||||
touching the source database, binding, configuration, or secrets. Table deletion cascades through
|
||||
columns and relationships; table-scoped relationship cleanup includes incoming and outgoing
|
||||
relationships. Curated and generated descriptions are editable; generated descriptions start null
|
||||
and Database Management can generate or consolidate them for selected tables, selected columns,
|
||||
all targets, or only targets whose Generated Description is missing.
|
||||
|
||||
Relationship Management is now reachable directly from each configured Fleet database. One
|
||||
Relationship Map shows read-only Physical Relationships together with Generated and Manual Logical
|
||||
Relationships, with Active, Excluded, and All filters. Administrators can add a single-column
|
||||
relationship, run deterministic name/PK/type inference, exclude or restore a logical relationship,
|
||||
or delete it permanently. Exclusion retains a tombstone that a rebuild cannot reactivate; permanent
|
||||
deletion allows a later rebuild to infer the same endpoints again. Inference uses no LLM, embedding,
|
||||
or source values. It supports normalized table-qualified names, unique non-generic PK names,
|
||||
composite-PK source columns, and the `*time_key -> dim_time.<single PK>` warehouse convention while
|
||||
ignoring bare generic names. Explicit table/column metadata cleanup remains a destructive boundary: it removes
|
||||
the attached logical relationships and exclusions and requires a full schema synchronization before
|
||||
inference or runtime publication can continue.
|
||||
|
||||
The previous Database Management renderer remains a temporary comparison fallback for development
|
||||
and staging only: `?db-ui=legacy` is honored in Vite development or when
|
||||
`VITE_DB_MANAGEMENT_LEGACY=true`; it is not a production presentation. The standalone Fleet Ledger
|
||||
prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
|
||||
after which both migration aids can be removed.
|
||||
|
||||
Schema refresh is one durable asynchronous engine with database-table, selected-table-column,
|
||||
relationship, and full-database actions. Database-level menus expose only the table, relationship,
|
||||
and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections
|
||||
also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
|
||||
one-active-operation-per-database exclusion with synchronization. Runs have leases and
|
||||
restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
|
||||
apply, retained history, and a live SSE log with polling fallback. Null metadata renders blank
|
||||
rather than as a placeholder.
|
||||
|
||||
Direct PostgreSQL and strict known-host-verified OpenSSH use `pg_catalog`. REST bindings use the
|
||||
typed full-snapshot `POST /rpc/schema_snapshot` contract when available. Servers such as the
|
||||
current PSD endpoint that exposes only `POST /rpc/run_query` use one catalog-owned read-only query
|
||||
to return the exact same strict v1 snapshot in a single round trip. Both paths remain fail-closed:
|
||||
an absent capability, query error, partial result, or invalid snapshot applies no catalog changes.
|
||||
SSH is not yet enabled for NL→SQL session runtime.
|
||||
|
||||
The catalog runs in the internal `catalog-db` PostgreSQL service. Kysely migrations are an explicit
|
||||
one-shot `catalog-migrate` operation; `scripts/run-stack.sh` runs it before local startup. Runtime
|
||||
sessions now consume the Catalog's active effective relationship map through an immutable JSON
|
||||
snapshot tied to the runtime-config lease. The harness uses that snapshot as its exclusive
|
||||
relationship source while retaining Git-pinned annotations for descriptive metadata; legacy
|
||||
runtimes without a snapshot keep the previous merge behavior. The accepted design is recorded in
|
||||
`docs/plans/2026-08-26-metadata-catalog-from-thothai.md`, the snapshot contract under
|
||||
`docs/contracts/`, and ADRs 0001–0012.
|
||||
|
||||
Semantic aliases, value descriptions, synonyms, and concepts remain deferred to their dedicated
|
||||
slices.
|
||||
|
||||
AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to
|
||||
`false`, including for newly synchronized columns. An administrator may request a local sensitivity
|
||||
analysis for one selected database, selected tables, or selected columns. One deterministic
|
||||
TypeScript classifier combines metadata, bounded source-content rules, and optional CPU-only NER;
|
||||
no generative model decides the result. Its `sensitive` or `non_sensitive` assessments remain an
|
||||
unsaved draft until the human reviews and saves any chosen flag changes, including a downgrade to
|
||||
non-sensitive. Coverage is reported separately; interrupted history may count unprocessed columns.
|
||||
Each started analysis records a separate Sensitivity Analysis Run with aggregate counters and safe
|
||||
ordered events. The progress drawer opens before the synchronous request completes, polls the run,
|
||||
and displays sanitized source-scan and local-NER phase/batch activity while classification is in
|
||||
progress. This operational history never stores per-column assessments, source values,
|
||||
matched spans, prompts, or free-form diagnostics. Saving a sensitive decision persists a sanitized
|
||||
Sensitivity Reason as column Catalog Metadata alongside the human-owned flag; clearing the flag
|
||||
clears that reason. Reloading still discards an unsaved review draft.
|
||||
For unprotected columns, up to five source rows and five representative non-null values may be sent
|
||||
transiently to the configured model provider. Protected columns are omitted from source reads and
|
||||
replaced in the prompt by deterministic plausible values derived only from their metadata. Existing
|
||||
descriptions are not regenerated when a flag changes.
|
||||
|
||||
The accepted AI-description design is recorded in
|
||||
`docs/plans/2026-08-28-ai-catalog-description-generation.md`, with the formal specification in the
|
||||
adjacent `-spec.md` document and Gitea issue #4. Gitea issues #5–#11 deliver the implementation.
|
||||
The runtime deliberately keeps ThothAI's simple operating model: one installation-wide sequential
|
||||
run owned by the backend, one short-lived Python/LiteLLM completion helper per request, and
|
||||
persistence limited to the run, its safe ordered text events, and each Generated Description as
|
||||
soon as it succeeds. The helper performs at most one provider retry and never falls back to another
|
||||
model. Stop terminates the current helper and retains prior results; three consecutive exhausted
|
||||
technical batches fail the run. Startup marks stale queued/running work interrupted, and Unlock is
|
||||
available only when no local start, worker, or helper is live. Runs remain inspectable through a
|
||||
live SSE log with ordered polling fallback; there is no automatic resume or user-facing generation
|
||||
CLI. ADRs 0009–0010 record the runtime and source-sampling decisions.
|
||||
|
||||
The Installation Model Catalog accepts the protected `DEEPSEEK_API_KEY` and `ZAI_API_KEY`
|
||||
references for metadata-generation providers.
|
||||
It also accepts a model with no secret reference only when its OpenAI-compatible endpoint is
|
||||
explicit; this covers the VPN-only AritmoLab Qwen 3.6 server without creating a fake operator
|
||||
credential. The Python client supplies only its fixed non-secret compatibility placeholder.
|
||||
The AritmoLab entry also sets `disableThinking: true`, mapped to the endpoint's chat-template flag,
|
||||
because its default reasoning prose would violate the worker's exact JSON response contract.
|
||||
|
||||
Logical relationship integration with core schema-linking is complete: session creation and resume
|
||||
materialize the active physical/generated/manual map, retrieval-pack generation and Pi receive the
|
||||
same runtime config, and snapshot validation fails closed on a declared missing, invalid, or orphaned
|
||||
endpoint. Broader publication of other Catalog metadata to schema-linking remains a separate future
|
||||
slice.
|
||||
|
||||
**Deferred follow-up — Sensitive Data Policy in schema-linking.** The policy is first delivered
|
||||
and tested in catalog description generation. Its enforcement for core schema-linking remains
|
||||
out of scope until the current tickets are closed and the owner has completed the acceptance test.
|
||||
At that gate, resume the design: `tht` must receive a read-only projection of the current Sensitive
|
||||
Data Flags and exclude values from columns marked sensitive from every LSH result before it is
|
||||
given to Pi. Do not start this integration before the owner gives final approval after that test.
|
||||
|
||||
## Active deployment work and manual gates
|
||||
|
||||
### PSD server deployment program
|
||||
|
||||
The approved design and executable entry point are:
|
||||
|
||||
- `docs/plans/2026-08-20-psd-server-deployment-program-design.md`
|
||||
- `docs/plans/2026-08-20-psd-server-deployment-program.md`
|
||||
- `docs/plans/2026-08-20-psd-server-survey.md`
|
||||
- `docs/plans/2026-08-20-psd-server-project-a-standalone.md`
|
||||
- `docs/plans/2026-08-20-psd-server-project-b-authentik.md`
|
||||
|
||||
Last recorded state:
|
||||
|
||||
- survey: `SURVEY_NO_GO`;
|
||||
- Project A: `BLOCKED_BY_SURVEY_AND_MUTATION_GATE`;
|
||||
- Project B: `BLOCKED_BY_PROJECT_A_AND_PRE_B_GATE`.
|
||||
|
||||
The deployment is a clean replacement: legacy sessions, indexes, and application configuration
|
||||
are not migration inputs. The existing stack remains intact until its documented mutation and
|
||||
rollback gates are explicitly approved. Shared Omics/LocalLLM networks, ETL Evidence, DWH,
|
||||
`dwh-auth`, Supabase, Authentik, Superset, and Aritmolab are outside cleanup scope.
|
||||
|
||||
Human acceptance guides and sanitized report templates live under `docs/testing/` and
|
||||
`docs/testing/evidence/`. The remediation checklist is
|
||||
`docs/operations/psd-server-survey-remediation-checklist.md`.
|
||||
|
||||
### Authentication
|
||||
|
||||
The local/OIDC authentication remediation passed its automated review on 2026-08-18. Release and
|
||||
PSD mutation gates remain governed by:
|
||||
|
||||
- `docs/architecture/authentication.md`;
|
||||
- `docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md`;
|
||||
- `docs/operations/psd-dwh-auth-rollout.md`;
|
||||
- `docs/testing/authentication-manual-acceptance.md`.
|
||||
|
||||
Do not infer authorization for server, Nginx, Authentik, database, credential, or cutover changes
|
||||
from an automated PASS.
|
||||
|
||||
## Verification status
|
||||
|
||||
- The P1.1 workspace-directory registry and P2–P6 preprocessing workstreams are implemented and
|
||||
have automated coverage.
|
||||
- Evidence restructuring has a real PSD acceptance PASS as recorded above.
|
||||
- AI Description Generation has automated coverage across installation setup, model selection,
|
||||
generation/consolidation scopes, bounded sampling, cancellation/recovery, history, SSE/polling,
|
||||
and the LiteLLM helper boundary.
|
||||
- L2 tests requiring real providers or remote databases remain opt-in.
|
||||
- Server deployment, release, and owner-operated acceptance steps remain pending wherever the
|
||||
referenced runbooks require explicit approval.
|
||||
|
||||
Run the layer-specific checks documented in `AGENTS.md`. For release-sensitive changes, also run
|
||||
the repository contract scripts in `scripts/` and build the MkDocs site.
|
||||
|
||||
## Operational invariants
|
||||
|
||||
- `tht`'s `-c`/`--config` option follows the subcommand; it is not a global option.
|
||||
- `--json` commands write pristine JSON to stdout.
|
||||
- Persisted phase documents and the decision ledger are the source of session truth; chat is not.
|
||||
- UI chrome is English; workspace document content retains the workspace language.
|
||||
- The backend refuses resume for finalized or archived sessions.
|
||||
- A resume must send `/riprendi-sessione <id>`; a new session must send `/nuova-domanda`.
|
||||
- DWH access is read-only.
|
||||
The [cleanup record](docs/maintenance/2026-09-15-documentation-cleanup.md) records
|
||||
retired sources and retained gates. Main contains source; Actions generates the
|
||||
`pages` branch. The live site requires the separate explicit deployment described
|
||||
in [public manual publication](docs/operations/public-docs-publication.md).
|
||||
|
||||
@@ -4,72 +4,51 @@ ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fasti
|
||||
core. The portable deployment runs two application services plus the installation-local metadata
|
||||
catalog; DWH and LLM services remain external. Semantic services are bundled in Compose.
|
||||
|
||||
Authentication is configured through the single host CLI tht: see the [local authentication guide](docs/install/authentication-local.md),
|
||||
[generic OIDC guide](docs/install/authentication-oidc.md), and [manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
|
||||
The same frontend supports **full** (its own header) and **embedded** (inside a
|
||||
portal). This choice is independent of authentication: the Mac uses full/local,
|
||||
Omics uses embedded/upstream with its existing login, and a standalone server
|
||||
can use full/OIDC. See [rendering architecture](docs/architecture/application-shell.md)
|
||||
and [configuration, Omics delivery and deploy](docs/operations/shell-and-localization.md).
|
||||
|
||||
## Docker Compose: local startup
|
||||
For the current server upgrade with Omics Portal, follow the ordered
|
||||
[Codex server handoff](docs/operations/server-codex-handoff.md), including source
|
||||
integration, embedded/upstream configuration, coordinated rollout and rollback.
|
||||
|
||||
Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`,
|
||||
`catalog-db`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external,
|
||||
configurable endpoints—even when they are co-located with ThothII.
|
||||
Local/OIDC authentication is configured through the host CLI `tht`; portal
|
||||
authentication is established by the trusted server proxy. See the
|
||||
[local guide](docs/install/authentication-local.md),
|
||||
[OIDC guide](docs/install/authentication-oidc.md),
|
||||
[upstream integration](docs/install/authentication-upstream.md), and
|
||||
[manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
|
||||
|
||||
From a fresh clone, run these commands from the repository root:
|
||||
For the clone-based manual standalone installation test on macOS, Windows, and Linux, use the
|
||||
[Italian procedure](docs/install/standalone-manual-it.md) or the
|
||||
[English procedure](docs/install/standalone-manual-en.md).
|
||||
|
||||
```sh
|
||||
cp deploy/env/local.env.example deploy/env/local.env
|
||||
# Copy docs/install/examples/thothii-installation.local.yaml to a protected operator path,
|
||||
# replace its placeholders, chmod it 600, and set that exact THT_INSTALLATION_CONFIG_SOURCE.
|
||||
# Edit deploy/env/local.env, including PI_AUTH_FILE, THT_SECRETS_FILE, and external endpoints.
|
||||
./scripts/run-stack.sh
|
||||
```
|
||||
The [public manual](https://git.tylconsulting.it/thothii-docs/) covers the product,
|
||||
installation, use and administration. Developer architecture, contracts, ADRs, tests,
|
||||
plans and release records remain in this repository but are excluded from MkDocs
|
||||
pages and search. This is an editorial boundary, not an access restriction on the
|
||||
public repository. See the [documentation cleanup review](docs/maintenance/2026-09-15-documentation-cleanup.md)
|
||||
for the executed consolidation and the inventory of historical sources retained in Git.
|
||||
|
||||
The launcher builds the core, starts `catalog-db`, runs the explicit one-shot Kysely migrations,
|
||||
then runs the base+local stack in the foreground. Migrations never run implicitly in backend
|
||||
startup. The core image contains its Pi runtime; no host `pi` executable is used. For a server
|
||||
installation, build the image, start the catalog, and run the same migration service before the
|
||||
application rollout:
|
||||
## Docker Compose and installation
|
||||
|
||||
```sh
|
||||
cp deploy/env/server.env.example deploy/env/server.env
|
||||
# Prepare a mode-600 thothii-installation.yaml from the server example and set its exact
|
||||
# path as THT_INSTALLATION_CONFIG_SOURCE. Edit all remaining storage/secret/endpoint paths.
|
||||
sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example build core
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example up -d catalog-db
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example run --rm catalog-migrate
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example up --build -d
|
||||
```
|
||||
For a fresh installation, follow the complete manual procedure in
|
||||
[Italian](docs/install/standalone-manual-it.md) or
|
||||
[English](docs/install/standalone-manual-en.md). Configure protected files first;
|
||||
then run the documented build, explicit migrations and startup commands with the
|
||||
same installation descriptor and Compose project. There is no installer or launcher.
|
||||
|
||||
The initializer is required for an empty or restored server Pi-state bind. It atomically creates
|
||||
the three regular targets hidden below the writable parent bind; protected Pi auth and tracked
|
||||
model/settings sources remain separate read-only mounts. See the server manual before substituting
|
||||
a root other than `/srv/thothii/pi-state`.
|
||||
The mandatory stack includes frontend, core, PostgreSQL catalog, Qdrant, Ollama
|
||||
and the embedding initializer. DWH and LLM endpoints remain external dependencies.
|
||||
Pi is included in the core image. Credentials and certificates belong in protected
|
||||
installation-local files, never in the workspace repository.
|
||||
|
||||
Workspace descriptors come from the Git remote configured by `THT_WORKSPACE_GIT_REMOTE`; their
|
||||
runtime endpoint and secret bindings remain installation-local. Open
|
||||
<http://127.0.0.1:8080> (set `THOTH_HTTP_PORT` in `deploy/env/local.env` to choose another
|
||||
loopback port).
|
||||
|
||||
Credentials and certificates are local protected files. Do not put them in environment examples,
|
||||
workspace YAML, URLs, or Compose interpolation values.
|
||||
|
||||
Application state is split across the named `settings`, `pi-state`, `workspace-registry`,
|
||||
`sessions`, `qdrant-data`, and `embedding-models` volumes. `docker compose down` keeps them.
|
||||
`qdrant-data` is a derived but persistent index store; `embedding-models` is an Ollama model
|
||||
cache for `qwen3-embedding:0.6b` with fixed `1024`-dimension embeddings. Only an explicit destructive command such as `docker compose
|
||||
down --volumes` removes them.
|
||||
|
||||
The frontend depends on the core health check and proxies `/health` and `/api/*` to it. The
|
||||
application health endpoint intentionally checks process readiness only; external dependency
|
||||
diagnostics are exposed by `tht doctor` and do not prevent the UI from starting.
|
||||
For developer topology, overlays and lifecycle details, see the internal
|
||||
[Compose reference](docs/operations/compose-reference.md). Ordinary stop/down keeps
|
||||
persistent data; removing volumes is destructive and is not an upgrade step.
|
||||
Process health is distinct from external dependency checks performed by doctor.
|
||||
|
||||
## Git-backed workspace repository
|
||||
|
||||
@@ -106,16 +85,18 @@ a remote user's partial list. The isolated deployment exercise is
|
||||
`./scripts/verify-workspace-install-docs.sh --profile local` or `--profile server`.
|
||||
|
||||
<!-- workspace-descriptor-contract:start -->
|
||||
Schema v4 is the only accepted workspace descriptor. Schema v1, v2, and v3 workspace descriptors
|
||||
are rejected before activation. Candidate snapshot validation therefore makes activation or a pull
|
||||
fail atomically while the prior valid snapshot remains active. One workspace owns one Qdrant collection;
|
||||
schema, Evidence, and Memory records share that collection and stay separated by indexed payload
|
||||
`kind`.
|
||||
Schema v4 is the only accepted workspace descriptor. It contains workspace identity and optional
|
||||
Evidence configuration only; PostgreSQL Metadata Catalog owns every database fact and binding.
|
||||
Schema v1, v2, and v3 descriptors are rejected before activation. Candidate snapshot validation
|
||||
therefore makes activation or a pull fail atomically while the prior valid snapshot remains active.
|
||||
Each workspace owns separate Qdrant `reference` and `memory` collections: Schema, relationships, and
|
||||
Evidence are replaceable reference data; Memory and solved questions have a persistent lifecycle.
|
||||
<!-- workspace-descriptor-contract:end -->
|
||||
|
||||
<!-- non-workspace-migration:start -->
|
||||
Convert a v3 descriptor before publication by setting `workspace.schema_version` to `4` and
|
||||
removing `llm_policy` and `semantic_index`; no database or Evidence field changes.
|
||||
Create a clean v4 descriptor containing only `workspace` and optional `evidence`. Do not copy the
|
||||
legacy database, diagnostics, `llm_policy`, or `semantic_index` blocks; configure the database in
|
||||
Database Management.
|
||||
<!-- non-workspace-migration:end -->
|
||||
|
||||
For NL→SQL runtime sessions, connector `ssh_tunnel` bindings remain diagnostic-only: their bounded
|
||||
@@ -223,15 +204,23 @@ job remains deterministic and does not claim Docker startup.
|
||||
|
||||
## Workspace preprocessing and S3 Evidence
|
||||
|
||||
Run preprocessing through the native host CLI and the installation descriptor:
|
||||
For an interactive run, select the workspace, expand **Administration** in the right sidebar, and
|
||||
use its **Preprocessing** control. The control explains any unmet prerequisite and exposes only the
|
||||
latest safe failure diagnostic. For unattended operation, use the native host CLI and installation
|
||||
descriptor:
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
|
||||
tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
|
||||
tht --installation /absolute/path/thothii-installation.yaml \
|
||||
workspace preprocess run --workspace <workspace-id>
|
||||
|
||||
tht --installation /absolute/path/thothii-installation.yaml \
|
||||
workspace preprocess clear --workspace <workspace-id>
|
||||
```
|
||||
|
||||
The CLI starts the profile-gated `workspace-maintenance` service and enforces the workspace,
|
||||
secret, Qdrant, and embedding contracts. See [Evidence](docs/evidence.md) and the
|
||||
The one-shot command starts the profile-gated `workspace-maintenance` service, reads database
|
||||
metadata from PostgreSQL, and rebuilds LSH plus schema/Evidence vectors. The clear command removes
|
||||
those derived artifacts while preserving the separate Memory collection. The core remains unavailable
|
||||
until preprocessing completes. See [Evidence](docs/evidence.md) and the
|
||||
[workspace preprocessing CLI contract](docs/contracts/workspace-preprocessing-cli.md).
|
||||
|
||||
S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance.
|
||||
@@ -347,6 +336,8 @@ The server profile stores sessions and per-user preferences directly in PostgreS
|
||||
dual write. Use [`deploy/compose.session-server.yaml.example`](deploy/compose.session-server.yaml.example)
|
||||
with the canonical base+server files and set `THT_SERVER_WORKSPACE_CONFIG` to an absolute,
|
||||
protected copy of [`deploy/workspaces/server-sessions.yaml.example`](deploy/workspaces/server-sessions.yaml.example).
|
||||
That file is an installation runtime template, not an authored workspace descriptor; database
|
||||
bindings are injected from the PostgreSQL Metadata Catalog for each runtime lease.
|
||||
|
||||
The runtime login needs membership in the no-login database role `thoth_sessions_runtime` only.
|
||||
The distinct, one-shot migrator login needs migration authority and uses
|
||||
|
||||
+44
-11
@@ -7,6 +7,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
import { createMemoryCleanup } from "./catalog/memory-cleanup.js";
|
||||
import { PiProcessManager } from "./pi/pi-process-manager.js";
|
||||
import { SseHub } from "./sse/sse-hub.js";
|
||||
import { authenticateSession, captureAuthConfigSnapshot, configuredOrigin } from "./auth/auth.js";
|
||||
@@ -76,6 +77,12 @@ import { CatalogLogicalRelationshipService } from "./catalog/logical-relationshi
|
||||
import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
|
||||
import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
|
||||
import { loadRuntimeModelCatalog, type RuntimeModelCatalog } from "./models/runtime-model-catalog.js";
|
||||
import { createProductionWorkspacePreprocessingService } from "./workspace-maintenance.js";
|
||||
import type { WorkspacePreprocessingService } from "./workspaces/preprocessing-service.js";
|
||||
import { PreprocessingStateStore } from "./workspaces/preprocessing-state.js";
|
||||
import { workspacePreprocessingRoutes } from "./routes/workspace-preprocessing.js";
|
||||
import { memoryRoutes } from "./routes/memory.js";
|
||||
import { evidenceRoutes } from "./routes/evidence.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
@@ -89,6 +96,7 @@ export interface BuildAppDeps {
|
||||
workspaceDiagnoser?: WorkspaceDiagnoser;
|
||||
workspaceDatabaseTester?: WorkspaceDatabaseTester;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear"> & Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
|
||||
catalogRepository?: CatalogRepository;
|
||||
catalogService?: CatalogService;
|
||||
catalogPostgresAccess?: CatalogPostgresAccess;
|
||||
@@ -156,6 +164,8 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
app.register(cookie);
|
||||
app.register(rateLimit, { global: false });
|
||||
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const tht = deps?.thtRunner ?? new ThtRunner({
|
||||
thtBin: config.thtBin,
|
||||
harnessDir: config.harnessDir,
|
||||
@@ -166,6 +176,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
secretsFile: config.secretsFile,
|
||||
secretFiles: config.secretFiles,
|
||||
workspaceSecretStore,
|
||||
catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
@@ -174,9 +185,15 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
});
|
||||
const workspacePreprocessingService = deps?.workspacePreprocessingService
|
||||
?? createProductionWorkspacePreprocessingService({
|
||||
config,
|
||||
catalogRepository,
|
||||
registry: workspaceRegistry,
|
||||
workspaceSecretStore,
|
||||
runner: tht as ThtRunner,
|
||||
});
|
||||
const hub = deps?.hub ?? new SseHub();
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
|
||||
const runtimeModelCatalog = deps?.runtimeModelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
|
||||
const mgr = deps?.mgr ?? new PiProcessManager(config, {
|
||||
@@ -242,14 +259,6 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
|
||||
const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
|
||||
?? new CatalogLogicalRelationshipService(catalogRepository);
|
||||
const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider
|
||||
?? (config.catalogDatabase === undefined
|
||||
? undefined
|
||||
: new EffectiveRelationshipSnapshotProvider(
|
||||
catalogRepository,
|
||||
catalogLogicalRelationshipService,
|
||||
catalogOperationCoordinator,
|
||||
));
|
||||
const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
|
||||
catalogPostgresAccess,
|
||||
workspaceSecretStore,
|
||||
@@ -259,6 +268,11 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
catalogSchemaIntrospector,
|
||||
catalogOperationCoordinator,
|
||||
config.catalogSyncTimeoutMs,
|
||||
createMemoryCleanup(tht as ThtRunner, {
|
||||
internalQdrantUrl: config.internalQdrantUrl, internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId, internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
}),
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
|
||||
@@ -457,7 +471,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
workspaceRuntimeSupport,
|
||||
modelCatalog: runtimeModelCatalog,
|
||||
maintenanceBarrier,
|
||||
effectiveRelationships,
|
||||
catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
|
||||
});
|
||||
app.post("/internal/maintenance/activate", async (req, reply) => {
|
||||
try {
|
||||
@@ -487,7 +501,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
return maintenanceBarrier.status();
|
||||
});
|
||||
sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
|
||||
memoryRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry, runtime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
} });
|
||||
metaRoutes(app, { harnessDir: config.harnessDir, modelCatalog: runtimeModelCatalog });
|
||||
evidenceRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry,
|
||||
registryRoot: config.workspaceRegistry.root, hostRegistryRoot: config.evidenceHostRegistryRoot,
|
||||
service: workspacePreprocessingService });
|
||||
workspaceRoutes(app, {
|
||||
registry: workspaceRegistry,
|
||||
config: config.workspaceRegistry,
|
||||
@@ -496,6 +519,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
secretStore: workspaceSecretStore,
|
||||
testDatabaseConnection: workspaceDatabaseTester,
|
||||
});
|
||||
workspacePreprocessingRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
registry: workspaceRegistry,
|
||||
service: workspacePreprocessingService,
|
||||
inputFingerprint: tht as ThtRunner,
|
||||
readLatestJob: (workspaceId) => new PreprocessingStateStore({
|
||||
dataRoot: config.dataRoot ?? "/data",
|
||||
workspaceId,
|
||||
}).readLatestJob(),
|
||||
});
|
||||
catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
|
||||
catalogTableRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
|
||||
@@ -119,7 +119,9 @@ export function authenticateSession(deps: AuthDependencies): preHandlerHookHandl
|
||||
subject: session.subject,
|
||||
...(session.displayName === undefined ? {} : { displayName: session.displayName }),
|
||||
roles: session.roles,
|
||||
permissions: session.permissions,
|
||||
// Sessions can outlive a deployment that changes the role permission catalog.
|
||||
// resolve() has already checked validity, including current local user roles.
|
||||
permissions: rolesToPermissions(session.roles),
|
||||
isAdmin: session.roles.includes("admin"),
|
||||
};
|
||||
if (STATE_CHANGING_METHODS.has(request.method)) {
|
||||
|
||||
@@ -38,7 +38,7 @@ const MAX_MAPPED_GROUPS = 128;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
export const PERMISSION_CATALOG: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
const invalid = (): Error => new Error("authentication configuration is invalid");
|
||||
|
||||
@@ -33,7 +33,7 @@ const EMPTY_HKDF_SALT = Buffer.alloc(0);
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
const PERMISSIONS = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
] as const satisfies readonly Permission[];
|
||||
|
||||
const invalid = (): Error => new Error("auth_session_store_invalid");
|
||||
|
||||
@@ -5,7 +5,7 @@ export type Role = "user" | "admin";
|
||||
export type Permission =
|
||||
| "session.use" | "session.read_all" | "session.manage_all"
|
||||
| "settings.manage" | "workspace.manage" | "workspace.secrets.manage"
|
||||
| "database.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
| "database.manage" | "memory.manage" | "evidence.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
|
||||
export interface AuthenticationSessionConfig {
|
||||
regularTtlSeconds: number;
|
||||
|
||||
@@ -59,10 +59,16 @@ const completionResponseSchema = z.object({
|
||||
class InvalidModelJsonError extends Error {}
|
||||
class InvalidModelSchemaError extends Error {}
|
||||
class MissingModelTargetsError extends Error {}
|
||||
|
||||
interface DescriptionGenerationFailureTarget {
|
||||
id: string;
|
||||
reference: string;
|
||||
}
|
||||
|
||||
class DescriptionGenerationBatchError extends Error {
|
||||
constructor(
|
||||
readonly failure: unknown,
|
||||
readonly failedTargets: readonly { id: string; label: "Catalog Column" | "Catalog Table" }[],
|
||||
readonly failedTargets: readonly DescriptionGenerationFailureTarget[],
|
||||
) {
|
||||
super("description generation batch failed");
|
||||
}
|
||||
@@ -138,7 +144,7 @@ type ParsedOutcome = z.infer<typeof outcomeSchema>;
|
||||
|
||||
interface DescriptionGenerationPlan {
|
||||
columnTargets: SelectedColumnTarget[];
|
||||
tableIds: string[];
|
||||
tableTargets: Array<{ id: string; name: string }>;
|
||||
}
|
||||
|
||||
interface DescriptionGenerationCounters {
|
||||
@@ -164,10 +170,17 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
|
||||
};
|
||||
}
|
||||
|
||||
function failureTarget(target: SelectedTarget) {
|
||||
function targetReference(target: SelectedTarget): string {
|
||||
return target.kind === "column"
|
||||
? { id: target.column.id, label: "Catalog Column" as const }
|
||||
: { id: target.table.id, label: "Catalog Table" as const };
|
||||
? `Column ${JSON.stringify(`${target.table.name}.${target.column.name}`)}`
|
||||
: `Table ${JSON.stringify(target.table.name)}`;
|
||||
}
|
||||
|
||||
function failureTarget(target: SelectedTarget): DescriptionGenerationFailureTarget {
|
||||
return {
|
||||
id: target.kind === "column" ? target.column.id : target.table.id,
|
||||
reference: targetReference(target),
|
||||
};
|
||||
}
|
||||
|
||||
const NON_GENERATABLE_DESCRIPTION: Record<DescriptionGenerationRun["language"], string> = {
|
||||
@@ -681,7 +694,7 @@ function safeFailure(error: unknown): string {
|
||||
function batchFailureEvent(error: DescriptionGenerationBatchError): string {
|
||||
const summary = safeFailure(error);
|
||||
return error.failedTargets.length > 0
|
||||
? `${summary} Affected ${error.failedTargets[0]!.label} target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.id).join(", ")}.`
|
||||
? `${summary} Affected target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.reference).join(", ")}.`
|
||||
: summary;
|
||||
}
|
||||
|
||||
@@ -794,7 +807,7 @@ export class DescriptionGenerationWorker {
|
||||
scope === "selected_columns" ? "column" : "table",
|
||||
);
|
||||
}
|
||||
const total = plan.columnTargets.length + plan.tableIds.length;
|
||||
const total = plan.columnTargets.length + plan.tableTargets.length;
|
||||
if (total === 0 && (scope === "all" || scope === "missing")) {
|
||||
throw new DescriptionGenerationNoEligibleTargetsError(scope);
|
||||
}
|
||||
@@ -934,12 +947,18 @@ export class DescriptionGenerationWorker {
|
||||
};
|
||||
await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
|
||||
throwIfCancelled(signal);
|
||||
if (plan.tableIds.length > 0) {
|
||||
const tableTargets = await this.resolveTableTargets(run.databaseId, plan.tableIds);
|
||||
if (plan.tableTargets.length > 0) {
|
||||
const tableTargets = await this.resolveTableTargets(
|
||||
run.databaseId,
|
||||
plan.tableTargets.map((target) => target.id),
|
||||
);
|
||||
if (!tableTargets) {
|
||||
throw new DescriptionGenerationBatchError(
|
||||
new Error("selected tables changed during generation"),
|
||||
plan.tableIds.map((id) => ({ id, label: "Catalog Table" })),
|
||||
plan.tableTargets.map((target) => ({
|
||||
id: target.id,
|
||||
reference: `Table ${JSON.stringify(target.name)}`,
|
||||
})),
|
||||
);
|
||||
}
|
||||
await this.processTargets(run, database, tableTargets, model, counters, signal);
|
||||
@@ -1093,8 +1112,8 @@ export class DescriptionGenerationWorker {
|
||||
run.id,
|
||||
"info",
|
||||
outcome.outcome === "generated"
|
||||
? `Generated description for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`
|
||||
: `Stored non-generatable result for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`,
|
||||
? `Generated description for ${targetReference(target)}.`
|
||||
: `Stored non-generatable result for ${targetReference(target)}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1188,16 +1207,22 @@ export class DescriptionGenerationWorker {
|
||||
}
|
||||
return {
|
||||
columnTargets,
|
||||
tableIds: tables
|
||||
tableTargets: tables
|
||||
.filter((table) => scope === "all" || !table.generatedDescription?.trim())
|
||||
.map((table) => table.id),
|
||||
.map((table) => ({ id: table.id, name: table.name })),
|
||||
};
|
||||
}
|
||||
if (scope === "selected_tables") {
|
||||
const tableById = new Map(tables.map((table) => [table.id, table]));
|
||||
const selected = targetIds.map((tableId) => tableById.get(tableId));
|
||||
if (selected.some((table) => table === undefined)) return undefined;
|
||||
return { columnTargets: [], tableIds: [...targetIds] };
|
||||
return {
|
||||
columnTargets: [],
|
||||
tableTargets: (selected as CatalogTable[]).map((table) => ({
|
||||
id: table.id,
|
||||
name: table.name,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const byId = new Map<string, SelectedColumnTarget>();
|
||||
@@ -1209,7 +1234,7 @@ export class DescriptionGenerationWorker {
|
||||
const targets = targetIds.map((columnId) => byId.get(columnId));
|
||||
return targets.some((target) => target === undefined)
|
||||
? undefined
|
||||
: { columnTargets: targets as SelectedColumnTarget[], tableIds: [] };
|
||||
: { columnTargets: targets as SelectedColumnTarget[], tableTargets: [] };
|
||||
}
|
||||
|
||||
private async resolveTableTargets(
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
|
||||
import type { CatalogSyncRun, WorkspaceDatabase } from "./types.js";
|
||||
|
||||
/** Internal continuation of an applied physical sync, using the harness Memory boundary. */
|
||||
export function createMemoryCleanup(runner: Pick<ThtRunner, "withPrincipal">, runtime: SemanticRuntimeConfig) {
|
||||
return async (database: WorkspaceDatabase, run: CatalogSyncRun): Promise<number> => {
|
||||
if (run.phase !== "memory_cleanup" || !run.plannedDiff) throw new Error("Physical cleanup is not committed");
|
||||
const result = await runner.withPrincipal({ issuer: "installation", subject: "catalog-sync",
|
||||
roles: ["admin"], permissions: ["memory.manage"], isAdmin: true,
|
||||
}).runWithRuntimeSnapshot(["memory", "admin", "--workspace", database.workspaceId], JSON.stringify({
|
||||
action: "cleanup", runtime, request: { sync_id: run.id, database: database.databaseName,
|
||||
schema_name: database.schema, removed_tables: run.plannedDiff.deletedTables,
|
||||
removed_columns: run.plannedDiff.deletedColumns.map(column => ({ table: column.tableName, column: column.columnName })),
|
||||
},
|
||||
}));
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.code !== 0 || payload.indexed !== true || !Number.isInteger(payload.deleted) || payload.deleted < 0) {
|
||||
throw new Error("Memory cleanup is incomplete");
|
||||
}
|
||||
return payload.deleted;
|
||||
};
|
||||
}
|
||||
@@ -18,6 +18,8 @@ import {
|
||||
type CatalogLogicalRelationshipCandidate,
|
||||
type CatalogLogicalRelationshipContext,
|
||||
type CatalogPhysicalRelationship,
|
||||
type CatalogPreprocessingStartResult,
|
||||
type CatalogPreprocessingClearResult,
|
||||
type CatalogSchemaDiff,
|
||||
type CatalogSyncCounts,
|
||||
type CatalogSyncEvent,
|
||||
@@ -72,6 +74,77 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
|
||||
return value ? clone(value) : undefined;
|
||||
}
|
||||
async beginPreprocessing(
|
||||
workspaceId: string,
|
||||
inputFingerprint: string,
|
||||
): Promise<CatalogPreprocessingStartResult> {
|
||||
const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
|
||||
if (!database) return { kind: "not_found" };
|
||||
if (database.preprocessingStatus === "running") return { kind: "already_running" };
|
||||
if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
|
||||
const catalogBusy = [...this.syncRuns.values()].some((run) =>
|
||||
run.databaseId === database.id
|
||||
&& ["queued", "running", "awaiting_confirmation", "applying"].includes(run.state))
|
||||
|| [...this.descriptionGenerationRuns.values()].some((run) =>
|
||||
run.databaseId === database.id && ["queued", "running"].includes(run.status))
|
||||
|| [...this.sensitivityAnalysisRuns.values()].some((run) =>
|
||||
run.databaseId === database.id && ["queued", "running"].includes(run.status));
|
||||
if (catalogBusy) return { kind: "catalog_busy" };
|
||||
const now = new Date().toISOString();
|
||||
const updated: WorkspaceDatabase = {
|
||||
...database,
|
||||
preprocessingStatus: "running",
|
||||
preprocessingInputFingerprint: inputFingerprint,
|
||||
preprocessedMetadataRevision: undefined,
|
||||
preprocessingStartedAt: now,
|
||||
preprocessingFinishedAt: undefined,
|
||||
preprocessingErrorCode: undefined,
|
||||
updatedAt: now,
|
||||
};
|
||||
this.records.set(database.id, updated);
|
||||
return { kind: "started", database: clone(updated) };
|
||||
}
|
||||
async finishPreprocessing(
|
||||
workspaceId: string,
|
||||
metadataContentRevision: number,
|
||||
inputFingerprint: string,
|
||||
outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
|
||||
): Promise<WorkspaceDatabase | undefined> {
|
||||
const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
|
||||
if (!database
|
||||
|| database.preprocessingStatus !== "running"
|
||||
|| database.metadataContentRevision !== metadataContentRevision
|
||||
|| database.preprocessingInputFingerprint !== inputFingerprint) return undefined;
|
||||
const updated: WorkspaceDatabase = {
|
||||
...database,
|
||||
preprocessingStatus: outcome.status,
|
||||
preprocessedMetadataRevision: outcome.status === "succeeded"
|
||||
? metadataContentRevision
|
||||
: undefined,
|
||||
preprocessingFinishedAt: new Date().toISOString(),
|
||||
preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : undefined,
|
||||
};
|
||||
this.records.set(database.id, updated);
|
||||
return clone(updated);
|
||||
}
|
||||
async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
|
||||
const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
|
||||
if (!database) return { kind: "not_found" };
|
||||
if (database.preprocessingStatus === "running") return { kind: "already_running" };
|
||||
const now = new Date().toISOString();
|
||||
const updated: WorkspaceDatabase = {
|
||||
...database,
|
||||
preprocessingStatus: "failed",
|
||||
preprocessingInputFingerprint: undefined,
|
||||
preprocessedMetadataRevision: undefined,
|
||||
preprocessingStartedAt: undefined,
|
||||
preprocessingFinishedAt: now,
|
||||
preprocessingErrorCode: "derived_data_cleared",
|
||||
updatedAt: now,
|
||||
};
|
||||
this.records.set(database.id, updated);
|
||||
return { kind: "cleared", database: clone(updated) };
|
||||
}
|
||||
async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
|
||||
if (databaseId !== undefined && !this.records.has(databaseId)) return undefined;
|
||||
|
||||
@@ -114,6 +187,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
connectionStatus: "untested",
|
||||
metadataContentRevision: 0,
|
||||
preprocessingStatus: "failed",
|
||||
};
|
||||
this.records.set(record.id, record);
|
||||
return clone(record);
|
||||
@@ -286,12 +361,24 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
return undefined;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
if (target === "database_columns") {
|
||||
const targets = [...this.columns.values()].filter((column) => (
|
||||
if (target === "database" || target === "database_columns") {
|
||||
const tableTargets = target === "database"
|
||||
? [...this.tables.values()].filter((table) => table.databaseId === databaseId)
|
||||
: [];
|
||||
const columnTargets = [...this.columns.values()].filter((column) => (
|
||||
this.tables.get(column.tableId)?.databaseId === databaseId
|
||||
));
|
||||
const copied = targets.filter((column) => Boolean(column.generatedDescription?.trim()));
|
||||
for (const column of copied) {
|
||||
const copiedTables = tableTargets.filter((table) => Boolean(table.generatedDescription?.trim()));
|
||||
const copiedColumns = columnTargets.filter((column) => Boolean(column.generatedDescription?.trim()));
|
||||
for (const table of copiedTables) {
|
||||
this.tables.set(table.id, {
|
||||
...table,
|
||||
description: table.generatedDescription,
|
||||
version: table.version + 1,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
for (const column of copiedColumns) {
|
||||
this.columns.set(column.id, {
|
||||
...column,
|
||||
description: column.generatedDescription,
|
||||
@@ -299,7 +386,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
return { copied: copied.length, skipped: targets.length - copied.length };
|
||||
const copied = copiedTables.length + copiedColumns.length;
|
||||
return { copied, skipped: tableTargets.length + columnTargets.length - copied };
|
||||
}
|
||||
if (selectedTargetIds.length === 0) return undefined;
|
||||
if (target === "tables") {
|
||||
@@ -838,6 +926,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined> {
|
||||
const database = this.records.get(databaseId);
|
||||
if (!database || database.version !== expectedDatabaseVersion) return undefined;
|
||||
@@ -1053,6 +1142,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
if (scope === "all") {
|
||||
this.records.set(databaseId, { ...database, schemaSyncedVersion: expectedDatabaseVersion, schemaSyncedAt: now });
|
||||
}
|
||||
if (syncRunId) await this.updateSyncRun(syncRunId, { phase: "memory_cleanup" });
|
||||
return {
|
||||
tables: (await this.listTables(databaseId)).length,
|
||||
columns: [...this.columns.values()].filter((column) => this.tables.get(column.tableId)?.databaseId === databaseId).length,
|
||||
@@ -1166,7 +1256,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
for (const run of this.syncRuns.values()) {
|
||||
if (["queued", "running", "awaiting_confirmation", "applying"].includes(run.state)) {
|
||||
await this.updateSyncRun(run.id, {
|
||||
state: "interrupted", phase: "completed", finishedAt: new Date().toISOString(),
|
||||
state: "interrupted", phase: run.phase === "memory_cleanup" ? "memory_cleanup" : "completed", finishedAt: new Date().toISOString(),
|
||||
errorCode: "SYNC_INTERRUPTED", errorMessage: "Synchronization was interrupted by a service restart",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -102,7 +102,7 @@ export function loadMetadataGenerationModels(options: {
|
||||
...(apiKeyEnv ? { apiKeyEnv, apiKey } : {}),
|
||||
}));
|
||||
}
|
||||
const result = new RestartLoadedMetadataGenerationModels(models, catalog.defaultMetadataGeneration);
|
||||
const result = new RestartLoadedMetadataGenerationModels(models, models.size ? catalog.defaultInteraction : null);
|
||||
const safe = result.catalog();
|
||||
return {
|
||||
catalog: () => ({
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import type {
|
||||
CatalogColumn,
|
||||
CatalogLogicalRelationship,
|
||||
CatalogPhysicalRelationship,
|
||||
CatalogRepository,
|
||||
CatalogTable,
|
||||
} from "./types.js";
|
||||
|
||||
export type CatalogDescriptionSource = "curated" | "generated" | "source_comment";
|
||||
|
||||
export interface CatalogMetadataSnapshotColumn {
|
||||
id: string;
|
||||
name: string;
|
||||
ordinalPosition: number;
|
||||
dataType: string;
|
||||
isNullable: boolean;
|
||||
defaultExpression: string | null;
|
||||
primaryKeyPosition: number | null;
|
||||
sensitive: boolean;
|
||||
description: string | null;
|
||||
descriptionSource: CatalogDescriptionSource | null;
|
||||
}
|
||||
|
||||
export interface CatalogMetadataSnapshotTable {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string | null;
|
||||
descriptionSource: CatalogDescriptionSource | null;
|
||||
columns: CatalogMetadataSnapshotColumn[];
|
||||
}
|
||||
|
||||
export interface CatalogMetadataSnapshotRelationship {
|
||||
id: string;
|
||||
origin: "physical" | "generated" | "manual";
|
||||
sourceTable: string;
|
||||
sourceColumns: string[];
|
||||
targetTable: string;
|
||||
targetColumns: string[];
|
||||
}
|
||||
|
||||
export interface CatalogMetadataSnapshot {
|
||||
schemaVersion: 1;
|
||||
workspaceId: string;
|
||||
databaseId: string;
|
||||
databaseName: string;
|
||||
schemaName: string;
|
||||
metadataContentRevision: number;
|
||||
tables: CatalogMetadataSnapshotTable[];
|
||||
relationships: CatalogMetadataSnapshotRelationship[];
|
||||
}
|
||||
|
||||
function effectiveDescription(value: {
|
||||
description: string | null;
|
||||
generatedDescription: string | null;
|
||||
sourceComment: string | null;
|
||||
}): { description: string | null; descriptionSource: CatalogDescriptionSource | null } {
|
||||
if (value.description?.trim()) {
|
||||
return { description: value.description.trim(), descriptionSource: "curated" };
|
||||
}
|
||||
if (value.generatedDescription?.trim()) {
|
||||
return { description: value.generatedDescription.trim(), descriptionSource: "generated" };
|
||||
}
|
||||
if (value.sourceComment?.trim()) {
|
||||
return { description: value.sourceComment.trim(), descriptionSource: "source_comment" };
|
||||
}
|
||||
return { description: null, descriptionSource: null };
|
||||
}
|
||||
|
||||
function snapshotColumn(column: CatalogColumn): CatalogMetadataSnapshotColumn {
|
||||
return {
|
||||
id: column.id,
|
||||
name: column.name,
|
||||
ordinalPosition: column.ordinalPosition,
|
||||
dataType: column.dataType,
|
||||
isNullable: column.isNullable,
|
||||
defaultExpression: column.defaultExpression,
|
||||
primaryKeyPosition: column.primaryKeyPosition,
|
||||
sensitive: column.sensitive,
|
||||
...effectiveDescription(column),
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotRelationship(
|
||||
relationship: CatalogPhysicalRelationship | CatalogLogicalRelationship,
|
||||
): CatalogMetadataSnapshotRelationship {
|
||||
const columns = [...relationship.columns].sort((left, right) => left.position - right.position);
|
||||
return {
|
||||
id: relationship.id,
|
||||
origin: relationship.origin,
|
||||
sourceTable: relationship.sourceTableName,
|
||||
sourceColumns: columns.map((column) => column.sourceColumnName),
|
||||
targetTable: relationship.targetTableName,
|
||||
targetColumns: columns.map((column) => column.targetColumnName),
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildCatalogMetadataSnapshot(
|
||||
repository: CatalogRepository,
|
||||
workspaceId: string,
|
||||
expectedMetadataContentRevision: number,
|
||||
): Promise<CatalogMetadataSnapshot> {
|
||||
const database = await repository.getByWorkspace(workspaceId);
|
||||
if (!database || database.preprocessingStatus !== "running") {
|
||||
throw new Error("catalog preprocessing lease is not active");
|
||||
}
|
||||
if (database.metadataContentRevision !== expectedMetadataContentRevision) {
|
||||
throw new Error("catalog metadata revision changed");
|
||||
}
|
||||
|
||||
const catalogTables = await repository.listTables(database.id);
|
||||
const tables: CatalogMetadataSnapshotTable[] = [];
|
||||
for (const table of [...catalogTables].sort((left, right) => left.name.localeCompare(right.name))) {
|
||||
const columns = await repository.listColumns(database.id, table.id);
|
||||
tables.push({
|
||||
id: table.id,
|
||||
name: table.name,
|
||||
...effectiveDescription(table),
|
||||
columns: columns
|
||||
.sort((left, right) => left.ordinalPosition - right.ordinalPosition || left.name.localeCompare(right.name))
|
||||
.map(snapshotColumn),
|
||||
});
|
||||
}
|
||||
|
||||
const physical = await repository.listRelationships(database.id);
|
||||
const logical = (await repository.listLogicalRelationships(database.id))
|
||||
.filter((relationship) => relationship.status === "active");
|
||||
const relationships = [...physical, ...logical]
|
||||
.map(snapshotRelationship)
|
||||
.sort((left, right) =>
|
||||
left.sourceTable.localeCompare(right.sourceTable)
|
||||
|| left.targetTable.localeCompare(right.targetTable)
|
||||
|| left.id.localeCompare(right.id));
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
workspaceId,
|
||||
databaseId: database.id,
|
||||
databaseName: database.databaseName,
|
||||
schemaName: database.schema,
|
||||
metadataContentRevision: expectedMetadataContentRevision,
|
||||
tables,
|
||||
relationships,
|
||||
};
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
|
||||
import * as canonicalModelIdsMigration from "./migrations/010_canonical_model_ids.js";
|
||||
import * as localSensitivityAnalysisMigration from "./migrations/011_local_sensitivity_analysis.js";
|
||||
import * as sensitivityReasonMigration from "./migrations/012_sensitivity_reason.js";
|
||||
import * as catalogPreprocessingStateMigration from "./migrations/013_catalog_preprocessing_state.js";
|
||||
|
||||
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
|
||||
const host = process.env.THT_CATALOG_DB_HOST;
|
||||
@@ -52,6 +53,7 @@ const provider: MigrationProvider = {
|
||||
"010_canonical_model_ids": canonicalModelIdsMigration,
|
||||
"011_local_sensitivity_analysis": localSensitivityAnalysisMigration,
|
||||
"012_sensitivity_reason": sensitivityReasonMigration,
|
||||
"013_catalog_preprocessing_state": catalogPreprocessingStateMigration,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.alterTable("workspace_databases")
|
||||
.addColumn("metadata_content_revision", "bigint", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("preprocessing_status", "text", (column) => column.notNull().defaultTo("failed"))
|
||||
.addColumn("preprocessing_input_fingerprint", "text")
|
||||
.addColumn("preprocessed_metadata_revision", "bigint")
|
||||
.addColumn("preprocessing_started_at", "timestamptz")
|
||||
.addColumn("preprocessing_finished_at", "timestamptz")
|
||||
.addColumn("preprocessing_error_code", "text")
|
||||
.execute();
|
||||
await sql`
|
||||
alter table workspace_databases
|
||||
add constraint workspace_databases_preprocessing_status_check
|
||||
check (preprocessing_status in ('running', 'succeeded', 'failed'))
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
create function catalog_metadata_write_guard()
|
||||
returns trigger
|
||||
language plpgsql
|
||||
as $$
|
||||
declare
|
||||
resolved_database_id uuid;
|
||||
current_status text;
|
||||
relation_id uuid;
|
||||
table_id uuid;
|
||||
begin
|
||||
if tg_table_name = 'catalog_tables' then
|
||||
resolved_database_id := coalesce(new.database_id, old.database_id);
|
||||
elsif tg_table_name = 'catalog_columns' then
|
||||
table_id := coalesce(new.table_id, old.table_id);
|
||||
select database_id into resolved_database_id from catalog_tables where id = table_id;
|
||||
elsif tg_table_name = 'catalog_relationships' then
|
||||
resolved_database_id := coalesce(new.database_id, old.database_id);
|
||||
elsif tg_table_name = 'catalog_relationship_columns' then
|
||||
relation_id := coalesce(new.relationship_id, old.relationship_id);
|
||||
select database_id into resolved_database_id from catalog_relationships where id = relation_id;
|
||||
elsif tg_table_name = 'catalog_logical_relationships' then
|
||||
resolved_database_id := coalesce(new.database_id, old.database_id);
|
||||
elsif tg_table_name = 'database_bindings' then
|
||||
if tg_op = 'UPDATE' and not (
|
||||
new.transport is distinct from old.transport
|
||||
or new.host is distinct from old.host
|
||||
or new.port is distinct from old.port
|
||||
or new.username is distinct from old.username
|
||||
or new.base_url is distinct from old.base_url
|
||||
or new.rest_path is distinct from old.rest_path
|
||||
or new.rest_auth is distinct from old.rest_auth
|
||||
or new.tls_servername is distinct from old.tls_servername
|
||||
or new.ssh_host is distinct from old.ssh_host
|
||||
or new.ssh_port is distinct from old.ssh_port
|
||||
or new.ssh_username is distinct from old.ssh_username
|
||||
or new.ssh_target_host is distinct from old.ssh_target_host
|
||||
or new.ssh_target_port is distinct from old.ssh_target_port
|
||||
) then
|
||||
return new;
|
||||
end if;
|
||||
resolved_database_id := coalesce(new.database_id, old.database_id);
|
||||
end if;
|
||||
if resolved_database_id is null then
|
||||
if tg_op = 'DELETE' then return old; else return new; end if;
|
||||
end if;
|
||||
|
||||
select preprocessing_status into current_status
|
||||
from workspace_databases
|
||||
where id = resolved_database_id
|
||||
for update;
|
||||
|
||||
if current_status = 'running' then
|
||||
raise exception 'catalog preprocessing is running'
|
||||
using errcode = '55000';
|
||||
end if;
|
||||
|
||||
update workspace_databases
|
||||
set metadata_content_revision = metadata_content_revision + 1,
|
||||
preprocessing_status = 'failed',
|
||||
preprocessing_finished_at = now(),
|
||||
preprocessing_error_code = 'catalog_changed',
|
||||
updated_at = now()
|
||||
where id = resolved_database_id;
|
||||
if tg_op = 'DELETE' then return old; else return new; end if;
|
||||
end;
|
||||
$$
|
||||
`.execute(db);
|
||||
|
||||
for (const table of [
|
||||
"catalog_tables",
|
||||
"catalog_columns",
|
||||
"catalog_relationships",
|
||||
"catalog_relationship_columns",
|
||||
"catalog_logical_relationships",
|
||||
"database_bindings",
|
||||
]) {
|
||||
await sql.raw(`
|
||||
create trigger ${table}_metadata_write_guard
|
||||
before insert or update or delete on ${table}
|
||||
for each row execute function catalog_metadata_write_guard()
|
||||
`).execute(db);
|
||||
}
|
||||
|
||||
await sql`
|
||||
create function catalog_database_configuration_guard()
|
||||
returns trigger
|
||||
language plpgsql
|
||||
as $$
|
||||
begin
|
||||
if new.workspace_id is distinct from old.workspace_id
|
||||
or new.engine is distinct from old.engine
|
||||
or new.database_name is distinct from old.database_name
|
||||
or new.schema_name is distinct from old.schema_name then
|
||||
if old.preprocessing_status = 'running' then
|
||||
raise exception 'catalog preprocessing is running'
|
||||
using errcode = '55000';
|
||||
end if;
|
||||
new.metadata_content_revision := old.metadata_content_revision + 1;
|
||||
new.preprocessing_status := 'failed';
|
||||
new.preprocessing_finished_at := now();
|
||||
new.preprocessing_error_code := 'catalog_changed';
|
||||
end if;
|
||||
return new;
|
||||
end;
|
||||
$$
|
||||
`.execute(db);
|
||||
await sql`
|
||||
create trigger workspace_databases_configuration_guard
|
||||
before update of workspace_id, engine, database_name, schema_name on workspace_databases
|
||||
for each row execute function catalog_database_configuration_guard()
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
create function catalog_operation_start_guard()
|
||||
returns trigger
|
||||
language plpgsql
|
||||
as $$
|
||||
declare
|
||||
current_status text;
|
||||
starting boolean;
|
||||
begin
|
||||
if tg_table_name = 'catalog_sync_runs' then
|
||||
starting := new.state in ('queued', 'running', 'awaiting_confirmation', 'applying');
|
||||
else
|
||||
starting := new.status in ('queued', 'running');
|
||||
end if;
|
||||
if not starting then
|
||||
return new;
|
||||
end if;
|
||||
|
||||
select preprocessing_status into current_status
|
||||
from workspace_databases
|
||||
where id = new.database_id
|
||||
for update;
|
||||
if current_status = 'running' then
|
||||
raise exception 'catalog preprocessing is running'
|
||||
using errcode = '55000';
|
||||
end if;
|
||||
return new;
|
||||
end;
|
||||
$$
|
||||
`.execute(db);
|
||||
|
||||
for (const table of [
|
||||
"catalog_sync_runs",
|
||||
"description_generation_runs",
|
||||
"sensitive_data_suggestion_runs",
|
||||
]) {
|
||||
await sql.raw(`
|
||||
create trigger ${table}_operation_start_guard
|
||||
before insert or update on ${table}
|
||||
for each row execute function catalog_operation_start_guard()
|
||||
`).execute(db);
|
||||
}
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
for (const table of [
|
||||
"catalog_sync_runs",
|
||||
"description_generation_runs",
|
||||
"sensitive_data_suggestion_runs",
|
||||
]) {
|
||||
await sql.raw(`drop trigger if exists ${table}_operation_start_guard on ${table}`).execute(db);
|
||||
}
|
||||
await sql`drop function if exists catalog_operation_start_guard()`.execute(db);
|
||||
await sql`drop trigger if exists workspace_databases_configuration_guard on workspace_databases`.execute(db);
|
||||
await sql`drop function if exists catalog_database_configuration_guard()`.execute(db);
|
||||
for (const table of [
|
||||
"catalog_tables",
|
||||
"catalog_columns",
|
||||
"catalog_relationships",
|
||||
"catalog_relationship_columns",
|
||||
"catalog_logical_relationships",
|
||||
"database_bindings",
|
||||
]) {
|
||||
await sql.raw(`drop trigger if exists ${table}_metadata_write_guard on ${table}`).execute(db);
|
||||
}
|
||||
await sql`drop function if exists catalog_metadata_write_guard()`.execute(db);
|
||||
await db.schema.alterTable("workspace_databases")
|
||||
.dropConstraint("workspace_databases_preprocessing_status_check").execute();
|
||||
for (const column of [
|
||||
"preprocessing_error_code",
|
||||
"preprocessing_finished_at",
|
||||
"preprocessing_started_at",
|
||||
"preprocessed_metadata_revision",
|
||||
"preprocessing_input_fingerprint",
|
||||
"preprocessing_status",
|
||||
"metadata_content_revision",
|
||||
]) {
|
||||
await sql.raw(`alter table workspace_databases drop column ${column}`).execute(db);
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,8 @@ import {
|
||||
type CatalogLogicalRelationshipCandidate,
|
||||
type CatalogLogicalRelationshipContext,
|
||||
type CatalogPhysicalRelationship,
|
||||
type CatalogPreprocessingStartResult,
|
||||
type CatalogPreprocessingClearResult,
|
||||
type CatalogSchemaDiff,
|
||||
type CatalogSyncCounts,
|
||||
type CatalogSyncEvent,
|
||||
@@ -54,6 +56,11 @@ import {
|
||||
} from "./types.js";
|
||||
|
||||
type Timestamp = ColumnType<Date, Date | string | undefined, Date | string>;
|
||||
type NullableTimestamp = ColumnType<
|
||||
Date | null,
|
||||
Date | string | null | undefined,
|
||||
Date | string | null
|
||||
>;
|
||||
|
||||
interface WorkspaceDatabaseTable {
|
||||
id: string;
|
||||
@@ -66,6 +73,13 @@ interface WorkspaceDatabaseTable {
|
||||
updatedAt: Timestamp;
|
||||
schemaSyncedVersion: number | null;
|
||||
schemaSyncedAt: Timestamp | null;
|
||||
metadataContentRevision: Generated<number>;
|
||||
preprocessingStatus: Generated<WorkspaceDatabase["preprocessingStatus"]>;
|
||||
preprocessingInputFingerprint: Generated<string | null>;
|
||||
preprocessedMetadataRevision: Generated<number | null>;
|
||||
preprocessingStartedAt: NullableTimestamp;
|
||||
preprocessingFinishedAt: NullableTimestamp;
|
||||
preprocessingErrorCode: Generated<string | null>;
|
||||
}
|
||||
|
||||
interface DatabaseBindingTable {
|
||||
@@ -325,6 +339,19 @@ function serialize(row: JoinedRow): WorkspaceDatabase {
|
||||
lastErrorMessage: present(row.lastErrorMessage),
|
||||
schemaSyncedVersion: present(row.schemaSyncedVersion),
|
||||
schemaSyncedAt: row.schemaSyncedAt == null ? undefined : new Date(row.schemaSyncedAt).toISOString(),
|
||||
metadataContentRevision: Number(row.metadataContentRevision),
|
||||
preprocessingStatus: row.preprocessingStatus,
|
||||
preprocessingInputFingerprint: present(row.preprocessingInputFingerprint),
|
||||
preprocessedMetadataRevision: row.preprocessedMetadataRevision == null
|
||||
? undefined
|
||||
: Number(row.preprocessedMetadataRevision),
|
||||
preprocessingStartedAt: row.preprocessingStartedAt == null
|
||||
? undefined
|
||||
: new Date(row.preprocessingStartedAt).toISOString(),
|
||||
preprocessingFinishedAt: row.preprocessingFinishedAt == null
|
||||
? undefined
|
||||
: new Date(row.preprocessingFinishedAt).toISOString(),
|
||||
preprocessingErrorCode: present(row.preprocessingErrorCode),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -477,6 +504,98 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
return id ? await this.get(id.id) : undefined;
|
||||
}
|
||||
|
||||
async beginPreprocessing(
|
||||
workspaceId: string,
|
||||
inputFingerprint: string,
|
||||
): Promise<CatalogPreprocessingStartResult> {
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases")
|
||||
.selectAll()
|
||||
.where("workspaceId", "=", workspaceId)
|
||||
.forUpdate()
|
||||
.executeTakeFirst();
|
||||
if (!database) return { kind: "not_found" };
|
||||
if (database.preprocessingStatus === "running") return { kind: "already_running" };
|
||||
if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
|
||||
|
||||
const activeSync = await trx.selectFrom("catalogSyncRuns")
|
||||
.select("id")
|
||||
.where("databaseId", "=", database.id)
|
||||
.where("state", "in", ["queued", "running", "awaiting_confirmation", "applying"])
|
||||
.executeTakeFirst();
|
||||
const activeDescriptions = await trx.selectFrom("descriptionGenerationRuns")
|
||||
.select("id")
|
||||
.where("databaseId", "=", database.id)
|
||||
.where("status", "in", ["queued", "running"])
|
||||
.executeTakeFirst();
|
||||
const activeSensitivity = await trx.selectFrom("sensitiveDataSuggestionRuns")
|
||||
.select("id")
|
||||
.where("databaseId", "=", database.id)
|
||||
.where("status", "=", "running")
|
||||
.executeTakeFirst();
|
||||
if (activeSync || activeDescriptions || activeSensitivity) return { kind: "catalog_busy" };
|
||||
|
||||
await trx.updateTable("workspaceDatabases")
|
||||
.set({
|
||||
preprocessingStatus: "running",
|
||||
preprocessingInputFingerprint: inputFingerprint,
|
||||
preprocessedMetadataRevision: null,
|
||||
preprocessingStartedAt: sql`now()`,
|
||||
preprocessingFinishedAt: null,
|
||||
preprocessingErrorCode: null,
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where("id", "=", database.id)
|
||||
.execute();
|
||||
return { kind: "started", database: (await selectOne(trx, database.id))! };
|
||||
});
|
||||
}
|
||||
|
||||
async finishPreprocessing(
|
||||
workspaceId: string,
|
||||
metadataContentRevision: number,
|
||||
inputFingerprint: string,
|
||||
outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
|
||||
): Promise<WorkspaceDatabase | undefined> {
|
||||
const result = await this.db.updateTable("workspaceDatabases")
|
||||
.set({
|
||||
preprocessingStatus: outcome.status,
|
||||
preprocessedMetadataRevision: outcome.status === "succeeded" ? metadataContentRevision : null,
|
||||
preprocessingFinishedAt: sql`now()`,
|
||||
preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : null,
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where("workspaceId", "=", workspaceId)
|
||||
.where("preprocessingStatus", "=", "running")
|
||||
.where("metadataContentRevision", "=", metadataContentRevision)
|
||||
.where("preprocessingInputFingerprint", "=", inputFingerprint)
|
||||
.returning("id")
|
||||
.executeTakeFirst();
|
||||
return result ? await this.get(result.id) : undefined;
|
||||
}
|
||||
|
||||
async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases")
|
||||
.select(["id", "preprocessingStatus"])
|
||||
.where("workspaceId", "=", workspaceId)
|
||||
.forUpdate()
|
||||
.executeTakeFirst();
|
||||
if (!database) return { kind: "not_found" };
|
||||
if (database.preprocessingStatus === "running") return { kind: "already_running" };
|
||||
await trx.updateTable("workspaceDatabases").set({
|
||||
preprocessingStatus: "failed",
|
||||
preprocessingInputFingerprint: null,
|
||||
preprocessedMetadataRevision: null,
|
||||
preprocessingStartedAt: null,
|
||||
preprocessingFinishedAt: sql`now()`,
|
||||
preprocessingErrorCode: "derived_data_cleared",
|
||||
updatedAt: sql`now()`,
|
||||
}).where("id", "=", database.id).execute();
|
||||
return { kind: "cleared", database: (await selectOne(trx, database.id))! };
|
||||
});
|
||||
}
|
||||
|
||||
async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
|
||||
const result = await sql<CatalogMetricsRow>`
|
||||
WITH requested_database AS (
|
||||
@@ -763,7 +882,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
targetIds: readonly string[],
|
||||
): Promise<CatalogDescriptionConsolidationCounts | undefined> {
|
||||
const selectedTargetIds = [...new Set(targetIds)];
|
||||
if (target !== "database_columns" && selectedTargetIds.length === 0) return undefined;
|
||||
if (target !== "database" && target !== "database_columns" && selectedTargetIds.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases").select("id")
|
||||
.where("id", "=", databaseId).forUpdate().executeTakeFirst();
|
||||
@@ -793,8 +914,23 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
};
|
||||
}
|
||||
|
||||
const tableRows = await trx.selectFrom("catalogTables").select("id")
|
||||
.where("databaseId", "=", databaseId).execute();
|
||||
const tableRows = await trx.selectFrom("catalogTables").select(["id", "generatedDescription"])
|
||||
.where("databaseId", "=", databaseId)
|
||||
.orderBy("id")
|
||||
.forUpdate()
|
||||
.execute();
|
||||
const copiedTableIds = target === "database"
|
||||
? tableRows
|
||||
.filter((row) => Boolean(row.generatedDescription?.trim()))
|
||||
.map((row) => row.id)
|
||||
: [];
|
||||
if (copiedTableIds.length > 0) {
|
||||
await trx.updateTable("catalogTables").set({
|
||||
description: sql`generated_description`,
|
||||
version: sql`version + 1`,
|
||||
updatedAt: sql`now()`,
|
||||
}).where("id", "in", copiedTableIds).execute();
|
||||
}
|
||||
const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
|
||||
.select(["id", "generatedDescription"])
|
||||
.where("tableId", "in", tableRows.map((table) => table.id))
|
||||
@@ -812,7 +948,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
version: sql`version + 1`,
|
||||
updatedAt: sql`now()`,
|
||||
});
|
||||
update = target === "database_columns"
|
||||
update = target === "database" || target === "database_columns"
|
||||
? update
|
||||
.where("tableId", "in", tableRows.map((table) => table.id))
|
||||
.where(sql<boolean>`nullif(btrim(generated_description), '') is not null`)
|
||||
@@ -820,8 +956,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
await update.execute();
|
||||
}
|
||||
return {
|
||||
copied: copiedIds.length,
|
||||
skipped: rows.length - copiedIds.length,
|
||||
copied: copiedTableIds.length + copiedIds.length,
|
||||
skipped: (target === "database" ? tableRows.length : 0) - copiedTableIds.length
|
||||
+ rows.length - copiedIds.length,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1467,6 +1604,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined> {
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases").select("version")
|
||||
@@ -1642,6 +1780,10 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
schemaSyncedAt: now,
|
||||
}).where("id", "=", databaseId).execute();
|
||||
}
|
||||
if (syncRunId) {
|
||||
await trx.updateTable("catalogSyncRuns").set({ phase: "memory_cleanup" })
|
||||
.where("id", "=", syncRunId).where("databaseId", "=", databaseId).execute();
|
||||
}
|
||||
return {
|
||||
tables: snapshot.tables.length,
|
||||
columns: scope === "tables" ? undefined : snapshot.columns.length,
|
||||
@@ -1745,7 +1887,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
|
||||
async interruptActiveSyncRuns(): Promise<void> {
|
||||
await this.db.updateTable("catalogSyncRuns").set({
|
||||
state: "interrupted", phase: "completed", errorCode: "worker_restarted",
|
||||
state: "interrupted", phase: sql`case when phase='memory_cleanup' then phase else 'completed' end`, errorCode: "worker_restarted",
|
||||
errorMessage: "Synchronization was interrupted by a backend restart.",
|
||||
finishedAt: sql`now()`, updatedAt: sql`now()`,
|
||||
leaseOwner: null, leaseExpiresAt: null,
|
||||
@@ -1835,6 +1977,9 @@ export class UnavailableCatalogRepository implements CatalogRepository {
|
||||
async list(): Promise<WorkspaceDatabase[]> { return this.fail(); }
|
||||
async get(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
|
||||
async getByWorkspace(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
|
||||
async beginPreprocessing(): Promise<CatalogPreprocessingStartResult> { return this.fail(); }
|
||||
async finishPreprocessing(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
|
||||
async clearPreprocessing(): Promise<CatalogPreprocessingClearResult> { return this.fail(); }
|
||||
async getCatalogMetrics(): Promise<CatalogMetrics | undefined> { return this.fail(); }
|
||||
async create(): Promise<WorkspaceDatabase> { return this.fail(); }
|
||||
async update(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { dirname } from "node:path";
|
||||
|
||||
import { resolveRuntimeBindings, type RuntimeBindings } from "../workspaces/bindings.js";
|
||||
import { buildInstallationContract, type InstallationSuffix } from "../workspaces/contracts.js";
|
||||
import {
|
||||
discoverWorkspaceSecretRequirements,
|
||||
} from "../workspaces/secret-requirements.js";
|
||||
import type {
|
||||
WorkspaceSecretMaterialization,
|
||||
WorkspaceSecretStore,
|
||||
} from "../workspaces/secret-store.js";
|
||||
import {
|
||||
validateWorkspaceDescriptor,
|
||||
type WorkspaceDescriptor,
|
||||
} from "../workspaces/schema.js";
|
||||
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||
import type { WorkspaceDatabase } from "./types.js";
|
||||
|
||||
export interface CatalogRuntimeBindingLease {
|
||||
workspace: WorkspaceDescriptor;
|
||||
bindings: RuntimeBindings;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
const CATALOG_SECRET_BY_SUFFIX: Readonly<Partial<Record<InstallationSuffix, string>>> = {
|
||||
PASSWORD_FILE: CATALOG_SECRET_IDS.password,
|
||||
API_KEY_FILE: CATALOG_SECRET_IDS.apiKey,
|
||||
TLS_CA_FILE: CATALOG_SECRET_IDS.tlsCa,
|
||||
SSH_PRIVATE_KEY_FILE: CATALOG_SECRET_IDS.sshPrivateKey,
|
||||
SSH_KNOWN_HOSTS_FILE: CATALOG_SECRET_IDS.sshKnownHosts,
|
||||
};
|
||||
|
||||
function runtimeWorkspace(
|
||||
workspace: WorkspaceDescriptor,
|
||||
database: WorkspaceDatabase,
|
||||
): WorkspaceDescriptor {
|
||||
if (workspace.workspace.id !== database.workspaceId) {
|
||||
throw new Error("Catalog database binding does not belong to the workspace");
|
||||
}
|
||||
const { dwh: _legacyDwh, diagnostics: _legacyDiagnostics, ...descriptor } = workspace;
|
||||
const binding = database.binding;
|
||||
return validateWorkspaceDescriptor({
|
||||
...descriptor,
|
||||
dwh: {
|
||||
engine: "postgres",
|
||||
database: database.databaseName,
|
||||
schema: database.schema,
|
||||
...(binding.port === undefined ? {} : { port: binding.port }),
|
||||
supported_transports: [binding.transport],
|
||||
},
|
||||
...(binding.transport === "rest_api" ? {
|
||||
diagnostics: {
|
||||
dwh_rest: {
|
||||
method: "GET",
|
||||
path: binding.restPath ?? "/health",
|
||||
auth: binding.restAuth ?? "bearer",
|
||||
response: { database: "database", schema: "schema" },
|
||||
},
|
||||
},
|
||||
} : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function setIfDefined(
|
||||
environment: NodeJS.ProcessEnv,
|
||||
name: string | undefined,
|
||||
value: string | number | undefined,
|
||||
): void {
|
||||
if (name !== undefined && value !== undefined && value !== "") environment[name] = String(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Project one PostgreSQL Catalog row into the legacy-shaped configuration consumed by the
|
||||
* Python runtime. The authored workspace remains database-free; this object exists only for
|
||||
* the lifetime of a backend-owned runtime lease.
|
||||
*/
|
||||
export function resolveCatalogRuntimeBinding(options: {
|
||||
workspace: WorkspaceDescriptor;
|
||||
database: WorkspaceDatabase;
|
||||
environment: NodeJS.ProcessEnv;
|
||||
secretRoots: readonly string[];
|
||||
secretStore: WorkspaceSecretStore;
|
||||
}): CatalogRuntimeBindingLease {
|
||||
const workspace = runtimeWorkspace(options.workspace, options.database);
|
||||
const evidenceRequirements = discoverWorkspaceSecretRequirements(
|
||||
options.workspace,
|
||||
options.environment,
|
||||
).filter(({ connector }) => connector === "evidence");
|
||||
const catalogSecretIds = Object.values(CATALOG_SECRET_IDS);
|
||||
let materialization: WorkspaceSecretMaterialization | undefined;
|
||||
try {
|
||||
materialization = options.secretStore.materialize(
|
||||
options.database.workspaceId,
|
||||
[...catalogSecretIds, ...evidenceRequirements.map(({ id }) => id)],
|
||||
);
|
||||
const environment: NodeJS.ProcessEnv = { ...options.environment };
|
||||
const roots = new Set(options.secretRoots);
|
||||
for (const path of materialization.files.values()) roots.add(dirname(path));
|
||||
|
||||
const variables = buildInstallationContract(workspace).variables;
|
||||
const variable = (role: "DWH" | "EVIDENCE", suffix: InstallationSuffix) => (
|
||||
variables.find((candidate) => candidate.role === role && candidate.suffix === suffix)?.name
|
||||
);
|
||||
const binding = options.database.binding;
|
||||
setIfDefined(environment, variable("DWH", "TRANSPORT"), binding.transport);
|
||||
setIfDefined(environment, variable("DWH", "HOST"), binding.host);
|
||||
setIfDefined(environment, variable("DWH", "PORT"), binding.port);
|
||||
setIfDefined(environment, variable("DWH", "BASE_URL"), binding.baseUrl);
|
||||
setIfDefined(environment, variable("DWH", "USER"), binding.username);
|
||||
setIfDefined(environment, variable("DWH", "SSH_HOST"), binding.sshHost);
|
||||
setIfDefined(environment, variable("DWH", "SSH_PORT"), binding.sshPort);
|
||||
setIfDefined(environment, variable("DWH", "SSH_USER"), binding.sshUsername);
|
||||
setIfDefined(environment, variable("DWH", "SSH_TARGET_HOST"), binding.sshTargetHost);
|
||||
setIfDefined(environment, variable("DWH", "SSH_TARGET_PORT"), binding.sshTargetPort);
|
||||
for (const [suffix, secretId] of Object.entries(CATALOG_SECRET_BY_SUFFIX) as Array<[
|
||||
InstallationSuffix,
|
||||
string,
|
||||
]>) {
|
||||
setIfDefined(environment, variable("DWH", suffix), materialization.files.get(secretId));
|
||||
}
|
||||
for (const requirement of evidenceRequirements) {
|
||||
setIfDefined(environment, requirement.variable, materialization.files.get(requirement.id));
|
||||
}
|
||||
|
||||
const bindings = resolveRuntimeBindings(workspace, environment, [...roots]);
|
||||
let released = false;
|
||||
return {
|
||||
workspace,
|
||||
bindings,
|
||||
release: () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
materialization?.release();
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
materialization?.release();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import { buildInstallationContract } from "../workspaces/contracts.js";
|
||||
import { resolveBinding } from "../workspaces/bindings.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../workspaces/schema.js";
|
||||
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
|
||||
@@ -45,60 +43,33 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
|
||||
secrets: Record<CatalogSecretName, boolean>;
|
||||
}
|
||||
|
||||
function bindingValue(workspace: WorkspaceDescriptor, values: Record<string, string>, suffix: string) {
|
||||
const variable = buildInstallationContract(workspace).variables.find((entry) => (
|
||||
entry.role === "DWH" && entry.suffix === suffix
|
||||
));
|
||||
return variable ? values[variable.name] : undefined;
|
||||
}
|
||||
|
||||
function numeric(value: string | undefined): number | undefined {
|
||||
if (!value) return undefined;
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65_535 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function yamlBinding(workspace: WorkspaceDescriptor, secretRoots: readonly string[]): DatabaseBinding {
|
||||
const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
|
||||
const value = (suffix: string) => bindingValue(workspace, effective.values, suffix);
|
||||
return {
|
||||
transport: effective.transport,
|
||||
host: value("HOST"),
|
||||
port: numeric(value("PORT")) ?? workspace.dwh.port,
|
||||
username: value("USER"),
|
||||
baseUrl: value("BASE_URL"),
|
||||
restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
|
||||
restAuth: workspace.diagnostics?.dwh_rest?.auth ?? "bearer",
|
||||
tlsServername: value("TLS_SERVERNAME"),
|
||||
sshHost: value("SSH_HOST"),
|
||||
sshPort: numeric(value("SSH_PORT")),
|
||||
sshUsername: value("SSH_USER"),
|
||||
sshTargetHost: value("SSH_TARGET_HOST"),
|
||||
sshTargetPort: numeric(value("SSH_TARGET_PORT")),
|
||||
};
|
||||
}
|
||||
|
||||
function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<CatalogSecretName, boolean> {
|
||||
return Object.fromEntries(Object.entries(CATALOG_SECRET_IDS).map(([name, id]) => (
|
||||
[name, store.has(workspaceId, id)]
|
||||
))) as Record<CatalogSecretName, boolean>;
|
||||
}
|
||||
|
||||
function workspaceRuntimeState(
|
||||
function databaseRuntimeState(
|
||||
store: WorkspaceSecretStore,
|
||||
workspace: WorkspaceDescriptor,
|
||||
secretRoots: readonly string[],
|
||||
database: WorkspaceDatabase,
|
||||
): NonNullable<CatalogListItem["runtimeBinding"]> {
|
||||
const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
|
||||
const secretVariables = new Set(requirements.map(({ variable }) => variable));
|
||||
const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
|
||||
const configurationRequired = requirements.some(({ id, required }) => (
|
||||
required && !store.has(workspace.workspace.id, id)
|
||||
)) || effective.missing.some((variable) => !secretVariables.has(variable));
|
||||
const { binding, workspaceId } = database;
|
||||
const configured = (id: string) => store.has(workspaceId, id);
|
||||
const connectionComplete = binding.transport === "postgres_direct"
|
||||
? Boolean(binding.host && binding.port && binding.username && configured(CATALOG_SECRET_IDS.password))
|
||||
: binding.transport === "rest_api"
|
||||
? Boolean(binding.baseUrl && (binding.restAuth === "none" || configured(CATALOG_SECRET_IDS.apiKey)))
|
||||
: Boolean(
|
||||
binding.username && binding.sshHost && binding.sshPort && binding.sshUsername
|
||||
&& binding.sshTargetHost && binding.sshTargetPort
|
||||
&& configured(CATALOG_SECRET_IDS.password)
|
||||
&& configured(CATALOG_SECRET_IDS.sshPrivateKey)
|
||||
&& configured(CATALOG_SECRET_IDS.sshKnownHosts),
|
||||
);
|
||||
return {
|
||||
transport: effective.transport,
|
||||
configurationState: configurationRequired ? "configuration_required" : "ready",
|
||||
sessionTransportSupported: effective.transport !== "ssh_tunnel",
|
||||
transport: binding.transport,
|
||||
configurationState: connectionComplete ? "ready" : "configuration_required",
|
||||
sessionTransportSupported: binding.transport !== "ssh_tunnel",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -145,17 +116,18 @@ export class CatalogService {
|
||||
const active = await Promise.all(workspaces.map(async (entry) => {
|
||||
const database = byWorkspace.get(entry.id);
|
||||
const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
|
||||
const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
|
||||
const base = database ?? {
|
||||
workspaceId: entry.id,
|
||||
engine: "postgres" as const,
|
||||
databaseName: workspace.dwh.database,
|
||||
schema: workspace.dwh.schema,
|
||||
databaseName: "",
|
||||
schema: "",
|
||||
version: 0,
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
binding: runtimeDatabaseBinding,
|
||||
binding: { transport: "postgres_direct" as const },
|
||||
connectionStatus: "untested" as const,
|
||||
metadataContentRevision: 0,
|
||||
preprocessingStatus: "failed" as const,
|
||||
};
|
||||
return {
|
||||
...base,
|
||||
@@ -167,7 +139,7 @@ export class CatalogService {
|
||||
blob: entry.revision.blob,
|
||||
},
|
||||
workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
|
||||
runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
|
||||
runtimeBinding: database ? databaseRuntimeState(this.secretStore, database) : null,
|
||||
configured: database !== undefined,
|
||||
secrets: secretState(this.secretStore, entry.id),
|
||||
};
|
||||
@@ -195,13 +167,13 @@ export class CatalogService {
|
||||
}
|
||||
|
||||
async normalizeInput(input: DatabaseConfigurationInput): Promise<DatabaseConfigurationInput> {
|
||||
const workspace = await this.ensureWorkspace(input.workspaceId);
|
||||
await this.ensureWorkspace(input.workspaceId);
|
||||
if (input.binding.transport !== "rest_api") return input;
|
||||
return {
|
||||
...input,
|
||||
binding: {
|
||||
...input.binding,
|
||||
restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
|
||||
restPath: input.binding.restPath ?? "/health",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -58,6 +58,8 @@ export class CatalogSyncWorker {
|
||||
private readonly introspector: CatalogSchemaIntrospector,
|
||||
private readonly operations: CatalogOperationCoordinator,
|
||||
private readonly timeoutMs: number,
|
||||
private readonly cleanupMemory: (database: WorkspaceDatabase, run: CatalogSyncRun) => Promise<number>
|
||||
= async () => 0,
|
||||
) {}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
@@ -67,6 +69,9 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
|
||||
async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise<CatalogSyncRun> {
|
||||
if ((await this.repository.listSyncRuns(database.id, 100)).some(run => run.phase === "memory_cleanup")) {
|
||||
throw new CatalogConflictError("Retry the pending Memory cleanup before starting another synchronization");
|
||||
}
|
||||
const uniqueTableIds = [...new Set(tableIds)];
|
||||
if (scope === "columns") {
|
||||
const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId)));
|
||||
@@ -109,7 +114,7 @@ export class CatalogSyncWorker {
|
||||
async cancel(runId: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = await this.repository.getSyncRun(runId);
|
||||
if (!run) return undefined;
|
||||
if (run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
|
||||
if (run.phase === "memory_cleanup" || run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
|
||||
await this.repository.requestSyncRunCancellation(runId);
|
||||
this.controllers.get(runId)?.abort();
|
||||
if (run.state === "queued" || run.state === "awaiting_confirmation") {
|
||||
@@ -138,6 +143,16 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
const database = await this.repository.get(previous.databaseId);
|
||||
if (!database) return undefined;
|
||||
if (previous.phase === "memory_cleanup") {
|
||||
const release = this.operations.reserve(database.id);
|
||||
try {
|
||||
const queued = await this.repository.updateSyncRun(runId, { state: "queued", cancelRequested: false,
|
||||
errorCode: null, errorMessage: null, finishedAt: null, leaseOwner: null, leaseExpiresAt: null });
|
||||
this.reservations.set(runId, release);
|
||||
this.launch(runId);
|
||||
return queued;
|
||||
} catch (error) { release(); throw error; }
|
||||
}
|
||||
return await this.start(database, previous.scope, previous.tableIds);
|
||||
}
|
||||
|
||||
@@ -179,6 +194,10 @@ export class CatalogSyncWorker {
|
||||
if (!database || database.version !== claimed.requestedDatabaseVersion) {
|
||||
throw new CatalogConflictError("Database binding changed before synchronization started");
|
||||
}
|
||||
if (claimed.phase === "memory_cleanup") {
|
||||
await this.finishMemoryCleanup(database, claimed);
|
||||
return;
|
||||
}
|
||||
const progress: CatalogSchemaScanProgress = async (phase, counts) => {
|
||||
await this.checkCancelled(runId);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
@@ -230,36 +249,30 @@ export class CatalogSyncWorker {
|
||||
claimed.scope,
|
||||
claimed.tableIds,
|
||||
snapshot,
|
||||
runId,
|
||||
);
|
||||
if (!applied) throw new CatalogConflictError("Database binding changed before schema changes were applied");
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: "succeeded",
|
||||
phase: "completed",
|
||||
counts: applied,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
confirmationToken: null,
|
||||
heartbeatAt: new Date().toISOString(),
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(runId, "info", "succeeded", "Synchronization completed.", { ...applied });
|
||||
this.release(runId);
|
||||
const cleanupRun = await this.repository.updateSyncRun(runId, { counts: applied });
|
||||
if (!cleanupRun) throw new Error("Synchronization run disappeared");
|
||||
await this.finishMemoryCleanup(database, cleanupRun);
|
||||
/* Completion is recorded only after the durable Memory cleanup succeeds. */
|
||||
return;
|
||||
} catch (error) {
|
||||
const current = await this.repository.getSyncRun(runId);
|
||||
const cancelled = !timedOut && (error instanceof SyncCancelledError || controller.signal.aborted || current?.cancelRequested);
|
||||
const failure = timedOut
|
||||
const failure = current?.phase === "memory_cleanup"
|
||||
? { code: "memory_cleanup_pending", message: "Catalog synchronized. Memory cleanup is pending; retry this synchronization to complete it." }
|
||||
: timedOut
|
||||
? { code: "schema_sync_timed_out", message: "Schema synchronization timed out." }
|
||||
: safeFailure(error);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: cancelled ? "cancelled" : "failed",
|
||||
phase: "completed",
|
||||
phase: current?.phase === "memory_cleanup" ? "memory_cleanup" : "completed",
|
||||
errorCode: cancelled ? null : failure.code,
|
||||
errorMessage: cancelled ? null : failure.message,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
observedSnapshot: current?.phase === "memory_cleanup" ? current.observedSnapshot : null,
|
||||
plannedDiff: current?.phase === "memory_cleanup" ? current.plannedDiff : null,
|
||||
confirmationToken: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
@@ -278,6 +291,20 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
}
|
||||
|
||||
private async finishMemoryCleanup(database: WorkspaceDatabase, run: CatalogSyncRun): Promise<void> {
|
||||
if (!run.plannedDiff || run.phase !== "memory_cleanup") throw new Error("Missing committed cleanup context");
|
||||
await this.repository.appendSyncEvent(run.id, "info", "memory_cleanup", "Removing Memory cards with deleted physical dependencies.");
|
||||
const memoryDeleted = await this.cleanupMemory(database, run);
|
||||
const counts = { ...run.counts, memoryDeleted };
|
||||
await this.repository.updateSyncRun(run.id, {
|
||||
state: "succeeded", phase: "completed", counts, finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null, plannedDiff: null, confirmationToken: null,
|
||||
heartbeatAt: new Date().toISOString(), leaseOwner: null, leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(run.id, "info", "succeeded", "Synchronization completed.", counts);
|
||||
this.release(run.id);
|
||||
}
|
||||
|
||||
private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void {
|
||||
const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const;
|
||||
for (const name of required) {
|
||||
|
||||
@@ -2,6 +2,7 @@ export const DATABASE_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"]
|
||||
export type DatabaseTransport = (typeof DATABASE_TRANSPORTS)[number];
|
||||
|
||||
export type ConnectionStatus = "untested" | "reachable" | "failed";
|
||||
export type CatalogPreprocessingStatus = "running" | "succeeded" | "failed";
|
||||
|
||||
export interface DatabaseBinding {
|
||||
transport: DatabaseTransport;
|
||||
@@ -36,8 +37,23 @@ export interface WorkspaceDatabase {
|
||||
lastErrorMessage?: string;
|
||||
schemaSyncedVersion?: number;
|
||||
schemaSyncedAt?: string;
|
||||
metadataContentRevision: number;
|
||||
preprocessingStatus: CatalogPreprocessingStatus;
|
||||
preprocessingInputFingerprint?: string;
|
||||
preprocessedMetadataRevision?: number;
|
||||
preprocessingStartedAt?: string;
|
||||
preprocessingFinishedAt?: string;
|
||||
preprocessingErrorCode?: string;
|
||||
}
|
||||
|
||||
export type CatalogPreprocessingStartResult =
|
||||
| { kind: "started"; database: WorkspaceDatabase }
|
||||
| { kind: "not_found" | "schema_stale" | "catalog_busy" | "already_running" };
|
||||
|
||||
export type CatalogPreprocessingClearResult =
|
||||
| { kind: "cleared"; database: WorkspaceDatabase }
|
||||
| { kind: "not_found" | "already_running" };
|
||||
|
||||
export interface CatalogMetrics {
|
||||
scope: "global" | "database";
|
||||
databaseId: string | null;
|
||||
@@ -196,7 +212,7 @@ export interface CatalogLogicalRelationshipCandidate {
|
||||
|
||||
export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
|
||||
export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
|
||||
export type CatalogDescriptionTarget = "tables" | "columns" | "database_columns";
|
||||
export type CatalogDescriptionTarget = "tables" | "columns" | "database" | "database_columns";
|
||||
|
||||
export interface CatalogMetadataDeleteCounts {
|
||||
tables: number;
|
||||
@@ -350,7 +366,7 @@ export type CatalogSyncState =
|
||||
export type CatalogSyncPhase =
|
||||
| "queued" | "connecting" | "scanning_tables" | "scanning_columns"
|
||||
| "scanning_relationships" | "planning" | "awaiting_confirmation"
|
||||
| "applying" | "completed";
|
||||
| "applying" | "memory_cleanup" | "completed";
|
||||
|
||||
export interface CatalogSchemaDiff {
|
||||
deletedTables: string[];
|
||||
@@ -359,6 +375,7 @@ export interface CatalogSchemaDiff {
|
||||
}
|
||||
|
||||
export interface CatalogSyncCounts {
|
||||
memoryDeleted?: number;
|
||||
tables?: number;
|
||||
columns?: number;
|
||||
relationships?: number;
|
||||
@@ -433,6 +450,17 @@ export interface CatalogRepository {
|
||||
list(): Promise<WorkspaceDatabase[]>;
|
||||
get(id: string): Promise<WorkspaceDatabase | undefined>;
|
||||
getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined>;
|
||||
beginPreprocessing(
|
||||
workspaceId: string,
|
||||
inputFingerprint: string,
|
||||
): Promise<CatalogPreprocessingStartResult>;
|
||||
finishPreprocessing(
|
||||
workspaceId: string,
|
||||
metadataContentRevision: number,
|
||||
inputFingerprint: string,
|
||||
outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
|
||||
): Promise<WorkspaceDatabase | undefined>;
|
||||
clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult>;
|
||||
getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined>;
|
||||
create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase>;
|
||||
update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined>;
|
||||
@@ -560,6 +588,7 @@ export interface CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined>;
|
||||
createSyncRun(
|
||||
databaseId: string,
|
||||
|
||||
@@ -30,8 +30,11 @@ export interface AppConfig {
|
||||
dataRoot?: string;
|
||||
ollamaEnsureTimeoutMs: number;
|
||||
piManagementTimeoutMs: number;
|
||||
/** Host CLI platform projected into Docker; not the browser or container OS. */
|
||||
hostPlatform?: string;
|
||||
secretsFile?: string;
|
||||
installationConfigFile?: string;
|
||||
evidenceHostRegistryRoot?: string;
|
||||
modelCatalogFile?: string;
|
||||
sensitivityNer?: {
|
||||
pythonExecutable: string;
|
||||
@@ -474,8 +477,10 @@ export function loadConfig(
|
||||
dataRoot: env.THT_DATA_ROOT,
|
||||
ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000),
|
||||
piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
|
||||
hostPlatform: env.THT_HOST_PLATFORM,
|
||||
secretsFile,
|
||||
installationConfigFile,
|
||||
evidenceHostRegistryRoot: env.THT_EVIDENCE_HOST_REGISTRY_ROOT || undefined,
|
||||
modelCatalogFile,
|
||||
sensitivityNer,
|
||||
piAuthFile,
|
||||
|
||||
@@ -41,15 +41,17 @@ const runtimeModelSchema = z.object({
|
||||
supportsReasoningEffort: z.boolean(),
|
||||
supportsStore: z.boolean(),
|
||||
maxTokensField: z.string().optional(),
|
||||
thinkingFormat: z.enum(["qwen", "qwen-chat-template"]).optional(),
|
||||
}).strict().optional(),
|
||||
}).strict().optional(),
|
||||
}).strict().refine((session) => !session.compatibility?.thinkingFormat || session.reasoning, {
|
||||
message: "thinkingFormat requires reasoning: true",
|
||||
}).optional(),
|
||||
metadataGeneration: z.object({ disableThinking: z.boolean() }).strict().optional(),
|
||||
}).strict();
|
||||
|
||||
const catalogSchema = z.object({
|
||||
schemaVersion: z.literal(1),
|
||||
defaultSession: canonicalId,
|
||||
defaultMetadataGeneration: canonicalId.optional(),
|
||||
schemaVersion: z.literal(2),
|
||||
defaultInteraction: canonicalId,
|
||||
embedding: z.object({ id: canonicalId, dimensions: z.number().int().positive() }).strict(),
|
||||
models: z.array(runtimeModelSchema).max(64),
|
||||
}).strict();
|
||||
@@ -57,8 +59,7 @@ const catalogSchema = z.object({
|
||||
export type RuntimeModel = z.infer<typeof runtimeModelSchema>;
|
||||
|
||||
export interface RuntimeModelCatalog {
|
||||
readonly defaultSession: string | null;
|
||||
readonly defaultMetadataGeneration: string | null;
|
||||
readonly defaultInteraction: string | null;
|
||||
readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
|
||||
sessionModels(): readonly RuntimeModel[];
|
||||
metadataModels(): readonly RuntimeModel[];
|
||||
@@ -66,19 +67,21 @@ export interface RuntimeModelCatalog {
|
||||
}
|
||||
|
||||
class RestartLoadedRuntimeModelCatalog implements RuntimeModelCatalog {
|
||||
readonly defaultSession: string | null;
|
||||
readonly defaultMetadataGeneration: string | null;
|
||||
readonly defaultInteraction: string | null;
|
||||
readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
|
||||
readonly #sessions: readonly RuntimeModel[];
|
||||
readonly #metadata: readonly RuntimeModel[];
|
||||
readonly #sessionIds: ReadonlySet<string>;
|
||||
|
||||
constructor(catalog?: z.infer<typeof catalogSchema>) {
|
||||
this.defaultSession = catalog?.defaultSession ?? null;
|
||||
this.defaultMetadataGeneration = catalog?.defaultMetadataGeneration ?? null;
|
||||
this.defaultInteraction = catalog?.defaultInteraction ?? null;
|
||||
this.embedding = catalog ? Object.freeze({ ...catalog.embedding }) : null;
|
||||
this.#sessions = Object.freeze((catalog?.models ?? []).filter((model) => model.session !== undefined));
|
||||
this.#metadata = Object.freeze((catalog?.models ?? []).filter((model) => model.metadataGeneration !== undefined));
|
||||
// One operational list. Session-only installations can still run Core, but once Admin
|
||||
// LLM models are configured every selectable model must support both adapters.
|
||||
const hasMetadata = catalog?.models.some((model) => model.metadataGeneration !== undefined);
|
||||
this.#sessions = Object.freeze((catalog?.models ?? []).filter((model) =>
|
||||
model.session !== undefined && (!hasMetadata || model.metadataGeneration !== undefined)));
|
||||
this.#metadata = Object.freeze(this.#sessions.filter((model) => model.metadataGeneration !== undefined));
|
||||
this.#sessionIds = new Set(this.#sessions.map((model) => model.id));
|
||||
}
|
||||
|
||||
@@ -129,11 +132,9 @@ export function loadRuntimeModelCatalog(file?: string): RuntimeModelCatalog {
|
||||
if (ids.size !== parsed.data.models.length) throw new Error("runtime model catalog contains duplicate models");
|
||||
const sessions = parsed.data.models.filter((model) => model.session !== undefined).map((model) => model.id);
|
||||
const metadata = parsed.data.models.filter((model) => model.metadataGeneration !== undefined).map((model) => model.id);
|
||||
if (!sessions.includes(parsed.data.defaultSession)) throw new Error("runtime model catalog session default is invalid");
|
||||
if ((metadata.length > 0) !== (parsed.data.defaultMetadataGeneration !== undefined)
|
||||
|| (parsed.data.defaultMetadataGeneration !== undefined
|
||||
&& !metadata.includes(parsed.data.defaultMetadataGeneration))) {
|
||||
throw new Error("runtime model catalog metadata default is invalid");
|
||||
if (!sessions.includes(parsed.data.defaultInteraction)
|
||||
|| (metadata.length > 0 && !metadata.includes(parsed.data.defaultInteraction))) {
|
||||
throw new Error("runtime model catalog interaction default is invalid");
|
||||
}
|
||||
return new RestartLoadedRuntimeModelCatalog(parsed.data);
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ import { loadSettings } from "./settings/settings-store.js";
|
||||
import { ThtRunner, type SessionRow } from "./tht/tht-runner.js";
|
||||
import { WorkspaceRegistry } from "./workspaces/registry.js";
|
||||
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
||||
import { createCatalogRepository } from "./catalog/repository.js";
|
||||
import type { CatalogRepository } from "./catalog/types.js";
|
||||
|
||||
type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status"
|
||||
| "session-inventory" | "workflow-doctor" | "workspace-integrity"
|
||||
@@ -30,7 +32,7 @@ const lifecyclePrincipal: PrincipalContext = {
|
||||
isAdmin: true,
|
||||
};
|
||||
|
||||
function operatorRunner(config: AppConfig): ThtRunner {
|
||||
function operatorRunner(config: AppConfig, catalogRepository: CatalogRepository): ThtRunner {
|
||||
const workspaceSecretStore = new WorkspaceSecretStore({
|
||||
root: config.workspaceSecretStoreRoot,
|
||||
runtimeRoot: config.workspaceSecretRuntimeRoot,
|
||||
@@ -46,6 +48,7 @@ function operatorRunner(config: AppConfig): ThtRunner {
|
||||
secretsFile: config.secretsFile,
|
||||
secretFiles: config.secretFiles,
|
||||
workspaceSecretStore,
|
||||
catalogRepository,
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
@@ -55,36 +58,55 @@ function operatorRunner(config: AppConfig): ThtRunner {
|
||||
}).withPrincipal(lifecyclePrincipal);
|
||||
}
|
||||
|
||||
async function withOperatorRunner<T>(
|
||||
config: AppConfig,
|
||||
operation: (runner: ThtRunner) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const catalogRepository = createCatalogRepository(config.catalogDatabase);
|
||||
try {
|
||||
return await operation(operatorRunner(config, catalogRepository));
|
||||
} finally {
|
||||
await catalogRepository.close?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function sessionInventory(config: AppConfig): Promise<Array<Pick<SessionRow, "status" | "archived">>> {
|
||||
const registry = new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const revisions = await registry.listRetainedSnapshots();
|
||||
const runner = operatorRunner(config);
|
||||
const sessions = new Map<string, SessionRow>();
|
||||
for (const revision of revisions) {
|
||||
for (const session of await runner.sessionList(revision.snapshotPath)) sessions.set(session.id, session);
|
||||
}
|
||||
return [...sessions.values()].map(({ status, archived }) => ({ status, archived: archived === true }));
|
||||
return await withOperatorRunner(config, async (runner) => {
|
||||
const sessions = new Map<string, SessionRow>();
|
||||
for (const revision of revisions) {
|
||||
for (const session of await runner.sessionList(revision.snapshotPath)) sessions.set(session.id, session);
|
||||
}
|
||||
return [...sessions.values()].map(({ status, archived }) => ({ status, archived: archived === true }));
|
||||
});
|
||||
}
|
||||
|
||||
async function workflowDiagnostics(config: AppConfig): Promise<{ ready: true; workspaces: number }> {
|
||||
const registry = new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const revisions = await registry.listRetainedSnapshots();
|
||||
if (revisions.length === 0) throw new Error("workflow diagnostics unavailable");
|
||||
const runner = operatorRunner(config);
|
||||
for (const revision of revisions) {
|
||||
const result = await runner.run(["doctor", "--json"], revision.snapshotPath);
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = JSON.parse(result.stdout);
|
||||
} catch {
|
||||
throw new Error("workflow diagnostics failed");
|
||||
return await withOperatorRunner(config, async (runner) => {
|
||||
for (const revision of revisions) {
|
||||
const runtime = await runner.acquireWorkspaceRuntime(revision.snapshotPath);
|
||||
try {
|
||||
const result = await runner.run(["doctor", "--json"], runtime.path);
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = JSON.parse(result.stdout);
|
||||
} catch {
|
||||
throw new Error("workflow diagnostics failed");
|
||||
}
|
||||
if (
|
||||
result.code !== 0 || !payload || typeof payload !== "object"
|
||||
|| (payload as { ok?: unknown }).ok !== true
|
||||
) throw new Error("workflow diagnostics failed");
|
||||
} finally {
|
||||
runtime.release();
|
||||
}
|
||||
}
|
||||
if (
|
||||
result.code !== 0 || !payload || typeof payload !== "object"
|
||||
|| (payload as { ok?: unknown }).ok !== true
|
||||
) throw new Error("workflow diagnostics failed");
|
||||
}
|
||||
return { ready: true, workspaces: revisions.length };
|
||||
return { ready: true, workspaces: revisions.length };
|
||||
});
|
||||
}
|
||||
|
||||
async function workspaceIntegrity(config: AppConfig): Promise<{
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
validateDeclarativePiConfig,
|
||||
} from "./managed-config.js";
|
||||
import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
|
||||
import { secretValue } from "../config/secret-bundle.js";
|
||||
|
||||
export interface PiModel {
|
||||
provider: string;
|
||||
@@ -64,6 +65,14 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModels
|
||||
}
|
||||
|
||||
const env = buildPiChildEnv({});
|
||||
// Pi's availability enumeration also needs the catalog-owned credentials for built-in
|
||||
// providers. It must keep working after their obsolete Pi auth entries are removed.
|
||||
for (const model of opts.modelCatalog?.sessionModels() ?? []) {
|
||||
const name = model.authentication.mode === "secret_env" ? model.authentication.apiKeyEnv : undefined;
|
||||
if (!name) continue;
|
||||
const value = secretValue(cfg, name);
|
||||
if (value) env[name] = value;
|
||||
}
|
||||
delete env.THT_DATA_ROOT;
|
||||
if (cfg.dataRoot !== undefined) env.THT_DATA_ROOT = cfg.dataRoot;
|
||||
const child = spawnFn(cfg.piBin, ["--mode", "rpc"], { cwd: cfg.harnessDir, env });
|
||||
|
||||
@@ -138,7 +138,7 @@ export interface PiRuntimeAgentSnapshot {
|
||||
* Bind a session Pi process to the exact managed auth/model bytes validated at spawn time.
|
||||
* Other agent resources remain live through symlinks, while session storage stays persistent.
|
||||
*/
|
||||
export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot {
|
||||
export function createPiRuntimeAgentSnapshot(options: { excludeAuthProvider?: string } = {}): PiRuntimeAgentSnapshot {
|
||||
const sourceAgentDir = configuredPiAgentDir();
|
||||
const auth = readPiAgentFile(sourceAgentDir, "auth.json", true);
|
||||
const models = readPiAgentFile(sourceAgentDir, "models.json", true);
|
||||
@@ -165,7 +165,18 @@ export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot {
|
||||
);
|
||||
}
|
||||
if (auth !== undefined) {
|
||||
writeFileSync(join(snapshotDir, "auth.json"), auth, { flag: "wx", mode: 0o600 });
|
||||
let effectiveAuth = auth;
|
||||
if (options.excludeAuthProvider) {
|
||||
const parsed = parsePiConfigJson(auth);
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new PiManagedConfigError();
|
||||
const provider = options.excludeAuthProvider.trim().toLowerCase();
|
||||
effectiveAuth = JSON.stringify(Object.fromEntries(
|
||||
Object.entries(parsed).filter(([key]) => key.trim().toLowerCase() !== provider),
|
||||
));
|
||||
}
|
||||
// secret_env is authoritative for this provider. Keep the operator's auth file intact,
|
||||
// but do not let an old Pi credential override the shared bundle inside this child.
|
||||
writeFileSync(join(snapshotDir, "auth.json"), effectiveAuth, { flag: "wx", mode: 0o600 });
|
||||
}
|
||||
if (models !== undefined) {
|
||||
writeFileSync(join(snapshotDir, "models.json"), models, { flag: "wx", mode: 0o600 });
|
||||
|
||||
@@ -36,6 +36,7 @@ export interface PiInstallationConfig {
|
||||
}
|
||||
|
||||
export interface PiStatus {
|
||||
hostPlatform: "linux" | "macos" | "windows";
|
||||
version?: string;
|
||||
ready: boolean;
|
||||
credentials: PiCredentialStatus;
|
||||
@@ -92,6 +93,9 @@ interface PiManagementDeps {
|
||||
}
|
||||
|
||||
export function createPiManagement(config: AppConfig, deps: PiManagementDeps): PiManagementService {
|
||||
const platform = config.hostPlatform ?? process.platform;
|
||||
const hostPlatform = platform === "darwin" ? "macos"
|
||||
: platform === "windows" || platform === "win32" ? "windows" : "linux";
|
||||
const now = deps.now ?? (() => new Date());
|
||||
const diagnostics: string[] = [];
|
||||
const addDiagnostic = (message: string): void => {
|
||||
@@ -106,23 +110,23 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
||||
});
|
||||
const credentialStatus = deps.credentialStatus ?? ((provider: string | undefined) => {
|
||||
try {
|
||||
const model = deps.modelCatalog.defaultSession
|
||||
? deps.modelCatalog.sessionModels().find((entry) => entry.id === deps.modelCatalog.defaultSession)
|
||||
const model = deps.modelCatalog.defaultInteraction
|
||||
? deps.modelCatalog.sessionModels().find((entry) => entry.id === deps.modelCatalog.defaultInteraction)
|
||||
: undefined;
|
||||
const credentialName = model?.authentication.mode === "secret_env"
|
||||
? model.authentication.apiKeyEnv
|
||||
: undefined;
|
||||
const configuredApiKey = configuredPiProviderApiKey(
|
||||
const configuredApiKey = credentialName ? `$${credentialName}` : configuredPiProviderApiKey(
|
||||
readConfiguredPiAgentFile("models.json", true),
|
||||
provider,
|
||||
) ?? (credentialName ? `$${credentialName}` : undefined);
|
||||
);
|
||||
return piProviderCredentialStatus({
|
||||
provider,
|
||||
authProviders: loadPiAuthProviders(),
|
||||
authProviders: credentialName ? new Set() : loadPiAuthProviders(),
|
||||
resolveCredentialValue: () => credentialName
|
||||
? secretValue(config, credentialName)
|
||||
: config.modelCatalogFile ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
|
||||
credentialFile: config.modelApiKeyFile,
|
||||
credentialFile: credentialName ? undefined : config.modelApiKeyFile,
|
||||
configuredApiKey,
|
||||
});
|
||||
} catch {
|
||||
@@ -152,8 +156,8 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
||||
const installationConfig = (): PiInstallationConfig => {
|
||||
const settings = readSettings();
|
||||
const reasoning = config.defaults.thinking ?? settings.thinking;
|
||||
const selected = deps.modelCatalog.defaultSession
|
||||
? splitCanonicalModelId(deps.modelCatalog.defaultSession)
|
||||
const selected = deps.modelCatalog.defaultInteraction
|
||||
? splitCanonicalModelId(deps.modelCatalog.defaultInteraction)
|
||||
: undefined;
|
||||
return {
|
||||
...(selected ? selected : {}),
|
||||
@@ -169,11 +173,11 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
||||
try {
|
||||
const currentVersion = await version();
|
||||
addDiagnostic("Pi version probe succeeded");
|
||||
return { version: currentVersion, ready: true, credentials, config: current, checkedAt };
|
||||
return { hostPlatform, version: currentVersion, ready: true, credentials, config: current, checkedAt };
|
||||
} catch (error) {
|
||||
const message = stableMessage(error, "Pi runtime is unavailable");
|
||||
addDiagnostic(message);
|
||||
return { ready: false, credentials, config: current, checkedAt, message };
|
||||
return { hostPlatform, ready: false, credentials, config: current, checkedAt, message };
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ export interface SessionRuntime {
|
||||
}
|
||||
|
||||
export interface RuntimeOptions {
|
||||
interactionLanguage?: string | null;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
thinking?: string;
|
||||
@@ -48,6 +49,7 @@ export class PiProcessManager {
|
||||
private spawnFn: (
|
||||
sessionId: string, author: string, provider: string | undefined, model: string | undefined,
|
||||
principal?: PrincipalContext, runtimeConfigPath?: string,
|
||||
interactionLanguage?: string | null,
|
||||
) => ChildProcessWithoutNullStreams;
|
||||
private loadAuthProviders: (agentDir: string) => ReadonlySet<string>;
|
||||
private modelCatalog: RuntimeModelCatalog;
|
||||
@@ -63,15 +65,15 @@ export class PiProcessManager {
|
||||
) {
|
||||
this.modelCatalog = opts?.modelCatalog ?? loadRuntimeModelCatalog(cfg.modelCatalogFile);
|
||||
this.modelCatalogConfigured = cfg.modelCatalogFile !== undefined
|
||||
|| this.modelCatalog.defaultSession !== null;
|
||||
|| this.modelCatalog.defaultInteraction !== null;
|
||||
this.loadAuthProviders = opts?.authProviders
|
||||
?? ((agentDir) => loadPiAuthProviders({ agentDir }));
|
||||
if (opts?.spawnFn) {
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
|
||||
this.spawnPi(opts.spawnFn!, sessionId, author, provider, model, principal, runtimeConfigPath);
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath, language) =>
|
||||
this.spawnPi(opts.spawnFn!, sessionId, author, provider, model, principal, runtimeConfigPath, language);
|
||||
} else {
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
|
||||
this.spawnPi(nodeSpawn, sessionId, author, provider, model, principal, runtimeConfigPath);
|
||||
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath, language) =>
|
||||
this.spawnPi(nodeSpawn, sessionId, author, provider, model, principal, runtimeConfigPath, language);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,33 +87,35 @@ export class PiProcessManager {
|
||||
private spawnPi(
|
||||
spawnFn: SpawnFn, sessionId: string, author: string, provider: string | undefined,
|
||||
model: string | undefined, principal?: PrincipalContext, runtimeConfigPath?: string,
|
||||
interactionLanguage?: string | null,
|
||||
): ChildProcessWithoutNullStreams {
|
||||
// This is the final shared boundary for createFor(), spawnFor(), and resume(). Validate
|
||||
// before auth-provider inspection, then make Pi consume the exact copied bytes rather than
|
||||
// reopening mutable mounted auth/models files after this check.
|
||||
const agent = createPiRuntimeAgentSnapshot();
|
||||
const catalogModel = provider && model
|
||||
? this.modelCatalog.sessionModels().find((entry) => entry.provider === provider && entry.model === model)
|
||||
: undefined;
|
||||
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
||||
? catalogModel.authentication.apiKeyEnv : undefined;
|
||||
const agent = createPiRuntimeAgentSnapshot({ excludeAuthProvider: credentialName ? provider : undefined });
|
||||
let child: ChildProcessWithoutNullStreams | undefined;
|
||||
try {
|
||||
const catalogModel = provider && model
|
||||
? this.modelCatalog.sessionModels()
|
||||
.find((entry) => entry.provider === provider && entry.model === model)
|
||||
: undefined;
|
||||
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
||||
? catalogModel.authentication.apiKeyEnv
|
||||
: undefined;
|
||||
const projectedApiKey = configuredPiProviderApiKey(agent.models, provider)
|
||||
?? (credentialName ? `$${credentialName}` : undefined);
|
||||
const projectedApiKey = credentialName ? `$${credentialName}`
|
||||
: configuredPiProviderApiKey(agent.models, provider);
|
||||
const env = buildPiChildEnv({
|
||||
provider,
|
||||
authProviders: this.loadAuthProviders(agent.agentDir),
|
||||
authProviders: credentialName ? new Set() : this.loadAuthProviders(agent.agentDir),
|
||||
credentialValue: credentialName
|
||||
? secretValue(this.cfg, credentialName)
|
||||
: this.modelCatalogConfigured ? undefined : secretValue(this.cfg, "THT_MODEL_API_KEY"),
|
||||
credentialFile: this.cfg.modelApiKeyFile,
|
||||
credentialFile: credentialName ? undefined : this.cfg.modelApiKeyFile,
|
||||
configuredApiKey: projectedApiKey,
|
||||
additions: { THT_SESSION: sessionId, THT_AUTHOR: author },
|
||||
});
|
||||
env.PI_CODING_AGENT_DIR = agent.agentDir;
|
||||
// A launch hint only: the gate reads the authoritative manifest before each turn.
|
||||
delete env.THT_INTERACTION_LANGUAGE;
|
||||
if (interactionLanguage) env.THT_INTERACTION_LANGUAGE = interactionLanguage;
|
||||
env.PI_CODING_AGENT_SESSION_DIR = agent.sessionDir;
|
||||
clearPrincipalEnvironment(env);
|
||||
if (principal) Object.assign(env, principalEnvironment(principal));
|
||||
@@ -189,7 +193,9 @@ export class PiProcessManager {
|
||||
const model = o.model ?? this.cfg.defaults.model;
|
||||
let child: ChildProcessWithoutNullStreams;
|
||||
try {
|
||||
child = this.spawnFn(sessionId, author, provider, model, o.principal, o.runtimeConfig?.path);
|
||||
child = this.spawnFn(
|
||||
sessionId, author, provider, model, o.principal, o.runtimeConfig?.path, o.interactionLanguage,
|
||||
);
|
||||
} catch (error) {
|
||||
o.runtimeConfig?.release();
|
||||
throw error;
|
||||
@@ -298,8 +304,13 @@ export class PiProcessManager {
|
||||
}
|
||||
|
||||
async resume(sessionId: string, tht: ThtRunner): Promise<SessionRuntime> {
|
||||
const manifest = await tht.sessionShow(sessionId) as { provider?: string; model?: string; thinking?: string } | null;
|
||||
const manifest = await tht.sessionShow(sessionId) as {
|
||||
provider?: string; model?: string; thinking?: string; interaction_language?: string | null;
|
||||
} | null;
|
||||
const language = manifest?.interaction_language
|
||||
?? (await tht.ensureInteractionLanguage(sessionId)).interaction_language;
|
||||
return this.spawnFor(sessionId, {
|
||||
interactionLanguage: language,
|
||||
provider: manifest?.provider,
|
||||
model: manifest?.model,
|
||||
thinking: manifest?.thinking,
|
||||
|
||||
@@ -70,28 +70,29 @@ export function createPiProviderSmoke(
|
||||
try {
|
||||
const canonicalProvider = canonicalPiProvider(provider);
|
||||
if (!canonicalProvider || timeoutMs <= 0) throw providerFailure();
|
||||
const configuredAuthProviders = authProviders();
|
||||
const configuredAuthProviders = new Set(authProviders());
|
||||
const configuredModels = options.readModelsStore
|
||||
? options.readModelsStore()
|
||||
: readConfiguredPiAgentFile("models.json", true);
|
||||
const catalog = options.modelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
|
||||
const catalogConfigured = config.modelCatalogFile !== undefined
|
||||
|| catalog.defaultSession !== null;
|
||||
|| catalog.defaultInteraction !== null;
|
||||
const catalogModel = catalog.sessionModels()
|
||||
.find((entry) => entry.provider === canonicalProvider && entry.model === model);
|
||||
const upstreamModel = catalogModel?.upstreamModel ?? model;
|
||||
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
||||
? catalogModel.authentication.apiKeyEnv
|
||||
: undefined;
|
||||
const projectedApiKey = configuredPiProviderApiKey(configuredModels, canonicalProvider)
|
||||
?? (credentialName ? `$${credentialName}` : undefined);
|
||||
if (credentialName) configuredAuthProviders.delete(canonicalProvider);
|
||||
const projectedApiKey = credentialName ? `$${credentialName}`
|
||||
: configuredPiProviderApiKey(configuredModels, canonicalProvider);
|
||||
const env = buildPiChildEnv({
|
||||
provider: canonicalProvider,
|
||||
authProviders: configuredAuthProviders,
|
||||
credentialValue: credentialName
|
||||
? secretValue(config, credentialName)
|
||||
: catalogConfigured ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
|
||||
credentialFile: config.modelApiKeyFile,
|
||||
credentialFile: credentialName ? undefined : config.modelApiKeyFile,
|
||||
configuredApiKey: projectedApiKey,
|
||||
});
|
||||
clearPrincipalEnvironment(env);
|
||||
|
||||
@@ -14,7 +14,7 @@ const consolidationSchema = z.discriminatedUnion("target", [
|
||||
target: z.enum(["tables", "columns"]),
|
||||
targetIds: z.array(idSchema).min(1).max(10_000),
|
||||
}).strict(),
|
||||
z.object({ target: z.literal("database_columns") }).strict(),
|
||||
z.object({ target: z.enum(["database", "database_columns"]) }).strict(),
|
||||
]);
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import path from "node:path";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { requirePermission, isPrincipalContext } from "../auth/authorization.js";
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
|
||||
|
||||
const params = z.object({ workspaceId: z.string().regex(/^[a-z][a-z0-9-]{2,62}$/),
|
||||
evidenceId: z.string().regex(/^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$/).optional() });
|
||||
const query = z.object({
|
||||
q: z.string().max(1000).optional(), kind: z.enum(["domain", "glossary", "enum", "example", "mapping", "normalization", "formula", "reference"]).optional(),
|
||||
purpose: z.enum(["disambiguation", "rewriting", "schema_linking", "sql_generation"]).optional(),
|
||||
status: z.enum(["new", "modified", "active", "removed", "review_required", "legacy", "invalid"]).optional(),
|
||||
concept: z.string().max(300).optional(), table: z.string().max(300).optional(), column: z.string().max(300).optional(),
|
||||
source: z.string().max(300).optional(), language: z.string().max(30).optional(),
|
||||
sort: z.enum(["title", "id", "kind", "status"]).optional(), direction: z.enum(["asc", "desc"]).optional(),
|
||||
page: z.coerce.number().int().positive().optional(), page_size: z.coerce.number().int().min(1).max(100).optional(),
|
||||
}).strict();
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
||||
|
||||
export function evidenceRoutes(app: FastifyInstance, deps: {
|
||||
runner: Pick<ThtRunner, "withPrincipal">; registry: Pick<WorkspaceRegistry, "list">;
|
||||
registryRoot: string; hostRegistryRoot?: string;
|
||||
service: Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
|
||||
}) {
|
||||
app.post("/workspaces/:workspaceId/evidence/sources", async (request, reply) => {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId } = params.parse(request.params);
|
||||
const body = z.discriminatedUnion("action", [
|
||||
z.object({ action: z.literal("refresh") }).strict(),
|
||||
z.object({ action: z.literal("decide"), sourceId: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
revision: z.string().regex(/^[a-f0-9]{64}$/), decision: z.enum(["keep", "replace"]) }).strict(),
|
||||
]).parse(request.body);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
if (!deps.service.evidenceSources) throw new Error("Evidence service unavailable");
|
||||
return await deps.service.evidenceSources({ workspaceId, ...body, actor: principal.subject });
|
||||
} catch (error) {
|
||||
return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
|
||||
message: error instanceof z.ZodError ? "Invalid source request." : "Evidence source service is unavailable." });
|
||||
}
|
||||
});
|
||||
app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence", read);
|
||||
app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence/:evidenceId", read);
|
||||
async function read(request: import("fastify").FastifyRequest, reply: import("fastify").FastifyReply) {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId, evidenceId } = params.parse(request.params);
|
||||
const filters = query.parse(request.query);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
const root = path.join(deps.registryRoot, "repo", workspaceId);
|
||||
const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
|
||||
["evidence", "admin", "--workspace", workspaceId],
|
||||
JSON.stringify({ root, query: { ...filters, ...(evidenceId ? { id: evidenceId } : {}) } }),
|
||||
);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.code !== 0) return reply.code(503).send(payload);
|
||||
if (evidenceId && !payload.item) return reply.code(404).send({ code: "evidence_not_found", message: "Evidence was not found." });
|
||||
const host = deps.hostRegistryRoot;
|
||||
const hostPath = host && (path.isAbsolute(host) || path.win32.isAbsolute(host))
|
||||
? (path.win32.isAbsolute(host) && !path.isAbsolute(host) ? path.win32 : path).join(host, "repo") : null;
|
||||
const hostJoin = hostPath && path.win32.isAbsolute(hostPath) && !path.isAbsolute(hostPath) ? path.win32.join : path.join;
|
||||
return { ...payload, location: { repository: hostPath, workspace: hostPath ? hostJoin(hostPath, workspaceId) : null,
|
||||
runtime_workspace: root, host: "Installation host", command: `tht workspace evidence consolidate --workspace ${workspaceId}`,
|
||||
git_commands: hostPath ? [
|
||||
`git -C ${quote(hostPath)} status --short -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} diff -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} add -A -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} commit --only -m 'Curate Evidence' -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} push`,
|
||||
] : [] } };
|
||||
} catch (error) {
|
||||
return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
|
||||
message: error instanceof z.ZodError ? "Evidence filters are invalid." : "Evidence archive is unavailable." });
|
||||
}
|
||||
}
|
||||
app.post("/workspaces/:workspaceId/evidence/consolidate", async (request, reply) => {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId } = params.parse(request.params);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
if (!deps.service.consolidateEvidence) throw new Error("Evidence service unavailable");
|
||||
return await deps.service.consolidateEvidence({ workspaceId });
|
||||
} catch { return reply.code(503).send({ code: "evidence_unavailable", message: "Evidence consolidation is unavailable." }); }
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
|
||||
|
||||
const paramsSchema = z.object({
|
||||
workspaceId: z.string().regex(/^[a-z][a-z0-9_-]{0,63}$/),
|
||||
cardId: z.string().regex(/^mem-[0-9a-f-]{36}$/).optional(),
|
||||
});
|
||||
const family = z.enum(["domain_clarification", "sql_rule", "solved_question", "explained_error"]);
|
||||
const cardSchema = z.object({
|
||||
family, subject: z.string().trim().min(1).max(1000),
|
||||
detail: z.string().max(50000).default(""), scope: z.string().trim().min(1).max(10000),
|
||||
rationale: z.string().max(10000).default(""), question: z.string().max(10000).default(""),
|
||||
sql: z.string().max(100000).default(""),
|
||||
concepts: z.array(z.string().trim().min(1).max(200)).max(100).default([]),
|
||||
dependencies: z.array(z.object({
|
||||
database: z.string().trim().min(1).max(200), schema_name: z.string().max(200).default(""),
|
||||
table: z.string().max(200).default(""), column: z.string().max(200).default(""),
|
||||
}).strict()).max(200).default([]),
|
||||
links: z.array(z.object({
|
||||
target_id: z.string().min(1).max(100), meaning: z.string().trim().min(1).max(1000),
|
||||
}).strict()).max(200).default([]),
|
||||
}).strict();
|
||||
const querySchema = z.object({
|
||||
q: z.string().max(1000).optional(), family: family.optional(),
|
||||
concept: z.string().max(200).optional(), database: z.string().max(200).optional(),
|
||||
table: z.string().max(200).optional(), column: z.string().max(200).optional(),
|
||||
origin: z.enum(["manual", "workflow"]).optional(),
|
||||
updated_after: z.iso.datetime({ offset: true }).optional(),
|
||||
updated_before: z.iso.datetime({ offset: true }).optional(),
|
||||
page: z.coerce.number().int().positive().optional(),
|
||||
page_size: z.coerce.number().int().min(1).max(100).optional(),
|
||||
sort: z.enum(["updated_at", "created_at", "subject", "family"]).optional(),
|
||||
direction: z.enum(["asc", "desc"]).optional(),
|
||||
}).strict();
|
||||
|
||||
export function memoryRoutes(app: FastifyInstance, deps: {
|
||||
runner: Pick<ThtRunner, "withPrincipal">;
|
||||
registry: Pick<WorkspaceRegistry, "list" | "read">;
|
||||
runtime: SemanticRuntimeConfig;
|
||||
}) {
|
||||
const invoke = (action: string) => async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const principal = requirePermission(request, reply, "memory.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId, cardId } = paramsSchema.parse(request.params);
|
||||
const input = action === "list" ? querySchema.parse(request.query)
|
||||
: action === "create" || action === "update"
|
||||
? { card: cardSchema.parse(request.body), ...(cardId ? { id: cardId } : {}) }
|
||||
: cardId ? { id: cardId } : {};
|
||||
// Registry identity is enough: administrative browsing must not require a DWH binding.
|
||||
const workspaces = await deps.registry.list();
|
||||
if (!workspaces.some(workspace => workspace.id === workspaceId)) {
|
||||
return reply.code(404).send({ code: "workspace_invalid", message: "Workspace was not found." });
|
||||
}
|
||||
const { workspace } = await deps.registry.read(workspaceId);
|
||||
const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
|
||||
["memory", "admin", "--workspace", workspaceId],
|
||||
JSON.stringify({ action, request: input, runtime: {
|
||||
...deps.runtime, memoryLanguage: workspace.workspace.language,
|
||||
} }),
|
||||
);
|
||||
let payload;
|
||||
try { payload = JSON.parse(result.stdout); } catch {
|
||||
return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
|
||||
}
|
||||
if (result.code !== 0) {
|
||||
const codes: Record<string, number> = {
|
||||
memory_invalid: 400, memory_forbidden: 403, memory_not_found: 404,
|
||||
memory_conflict: 409, memory_unavailable: 503,
|
||||
};
|
||||
const code = typeof payload?.code === "string" && payload.code in codes
|
||||
? payload.code : "memory_unavailable";
|
||||
return reply.code(codes[code]).send({ code, message: "Memory operation could not be completed." });
|
||||
}
|
||||
return reply.code(action === "create" ? 201 : 200).send(payload);
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({ code: "memory_invalid", message: "Memory request is invalid." });
|
||||
}
|
||||
return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
|
||||
}
|
||||
};
|
||||
app.get("/workspaces/:workspaceId/memory", invoke("list"));
|
||||
app.post("/workspaces/:workspaceId/memory", invoke("create"));
|
||||
app.get("/workspaces/:workspaceId/memory/pending", invoke("pending"));
|
||||
app.get("/workspaces/:workspaceId/memory/:cardId", invoke("show"));
|
||||
app.put("/workspaces/:workspaceId/memory/:cardId", invoke("update"));
|
||||
app.delete("/workspaces/:workspaceId/memory/:cardId", invoke("delete"));
|
||||
app.post("/workspaces/:workspaceId/memory/:cardId/retry", invoke("retry"));
|
||||
}
|
||||
+160
-31
@@ -11,8 +11,9 @@ import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js";
|
||||
import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
|
||||
import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js";
|
||||
import { splitCanonicalModelId, type RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
|
||||
import type { CatalogRepository } from "../catalog/types.js";
|
||||
import { interactionLanguage } from "../tht/interaction-language.js";
|
||||
|
||||
const BOOTSTRAP_FAILURE_MESSAGE =
|
||||
"Session startup failed. Check configuration and connectivity, then Resume the session.";
|
||||
@@ -42,12 +43,40 @@ export function sessionRoutes(
|
||||
/** Fail-closed installation/runtime transport capability check. */
|
||||
workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean;
|
||||
maintenanceBarrier: MaintenanceBarrier;
|
||||
/** Optional only for narrow route-test stubs and installations without a Catalog database. */
|
||||
effectiveRelationships?: EffectiveRelationshipSnapshotProvider;
|
||||
modelCatalog: RuntimeModelCatalog;
|
||||
/** PostgreSQL authority for mandatory preprocessing admission. */
|
||||
catalogRepository?: CatalogRepository;
|
||||
},
|
||||
) {
|
||||
const lifecycleTails = new Map<string, Promise<void>>();
|
||||
|
||||
const preprocessingIsCurrent = async (
|
||||
workspaceId: string,
|
||||
inputFingerprint?: string,
|
||||
): Promise<boolean> => {
|
||||
if (!d.catalogRepository) return true;
|
||||
const database = await d.catalogRepository.getByWorkspace(workspaceId);
|
||||
return database !== undefined
|
||||
&& database.preprocessingStatus === "succeeded"
|
||||
&& database.preprocessedMetadataRevision === database.metadataContentRevision
|
||||
&& (inputFingerprint === undefined
|
||||
|| database.preprocessingInputFingerprint === inputFingerprint);
|
||||
};
|
||||
|
||||
const catalogTransportSupportsSessionRuntime = async (workspaceId: string): Promise<boolean> => {
|
||||
if (!d.catalogRepository) return true;
|
||||
const database = await d.catalogRepository.getByWorkspace(workspaceId);
|
||||
return database === undefined || database.binding.transport !== "ssh_tunnel";
|
||||
};
|
||||
|
||||
const currentInputFingerprint = async (
|
||||
runner: any,
|
||||
workspaceConfigPath: string,
|
||||
): Promise<string | undefined> => (
|
||||
typeof runner.workspaceInputFingerprint === "function"
|
||||
? await runner.workspaceInputFingerprint(workspaceConfigPath)
|
||||
: undefined
|
||||
);
|
||||
const boundRuntimes = new Map<
|
||||
string,
|
||||
ReturnType<PiProcessManager["createFor"]>
|
||||
@@ -92,16 +121,13 @@ export function sessionRoutes(
|
||||
const optionsWithRuntimeConfig = async (
|
||||
runner: any,
|
||||
workspaceConfigPath: string | undefined,
|
||||
workspaceId: string | undefined,
|
||||
_workspaceId: string | undefined,
|
||||
options: any,
|
||||
) => {
|
||||
if (!workspaceConfigPath || typeof runner.acquireWorkspaceRuntime !== "function") return options;
|
||||
const effectiveRelationships = workspaceId && d.effectiveRelationships
|
||||
? await d.effectiveRelationships.render(workspaceId)
|
||||
: undefined;
|
||||
return {
|
||||
...options,
|
||||
runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath, effectiveRelationships),
|
||||
runtimeConfig: await runner.acquireWorkspaceRuntime(workspaceConfigPath),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -125,13 +151,34 @@ export function sessionRoutes(
|
||||
});
|
||||
app.addHook("onResponse", async (req) => { admissionLeases.get(req)?.(); });
|
||||
|
||||
type SessionRevisionScan = {
|
||||
revisions: Awaited<ReturnType<typeof d.workspaceRegistry.list>>;
|
||||
retainedComplete: boolean;
|
||||
};
|
||||
let retainedSnapshotWarning: string | null = null;
|
||||
|
||||
/** Include retained historical descriptors so removed workspaces remain resumable. */
|
||||
const sessionRevisions = async () => {
|
||||
const sessionRevisions = async (): Promise<SessionRevisionScan> => {
|
||||
const registry = d.workspaceRegistry as Partial<WorkspaceRegistry>;
|
||||
if (typeof registry.listRetainedSnapshots === "function") {
|
||||
return await registry.listRetainedSnapshots();
|
||||
try {
|
||||
const revisions = await registry.listRetainedSnapshots();
|
||||
retainedSnapshotWarning = null;
|
||||
return { revisions, retainedComplete: true };
|
||||
} catch (error) {
|
||||
// A legacy/corrupt historical snapshot must not make active sessions (and their SSE
|
||||
// reviewer gates) unreachable. The registry still rejects that snapshot; this fallback
|
||||
// exposes only descriptors from the verified active state and deliberately disables
|
||||
// retention reconciliation because the resulting session view is incomplete.
|
||||
const detail = error instanceof Error ? error.message : "unknown error";
|
||||
if (retainedSnapshotWarning !== detail) {
|
||||
console.warn("[sessions] retained snapshot discovery failed; using active snapshots:", detail);
|
||||
retainedSnapshotWarning = detail;
|
||||
}
|
||||
return { revisions: await d.workspaceRegistry.list(), retainedComplete: false };
|
||||
}
|
||||
}
|
||||
return await d.workspaceRegistry.list();
|
||||
return { revisions: await d.workspaceRegistry.list(), retainedComplete: true };
|
||||
};
|
||||
|
||||
const isNotFound = (error: unknown) =>
|
||||
@@ -175,7 +222,7 @@ export function sessionRoutes(
|
||||
};
|
||||
let revisions: Awaited<ReturnType<typeof d.workspaceRegistry.list>>;
|
||||
try {
|
||||
revisions = await sessionRevisions();
|
||||
({ revisions } = await sessionRevisions());
|
||||
} catch (registryError) {
|
||||
// Sessions created before revision pinning still live under the installation's legacy
|
||||
// default config. Keep that compatibility path available when a fresh installation has
|
||||
@@ -335,8 +382,17 @@ export function sessionRoutes(
|
||||
app.post("/sessions", async (req, reply) => {
|
||||
const b = req.body as {
|
||||
question: string; name?: string; workspace?: string; workspaceId?: string;
|
||||
provider?: string; model?: string; thinking?: string;
|
||||
provider?: string; model?: string; thinking?: string; interactionLanguage?: unknown;
|
||||
};
|
||||
const language = interactionLanguage(b?.interactionLanguage);
|
||||
if (!language) return reply.code(400).send({
|
||||
code: "invalid_interaction_language",
|
||||
error: "interactionLanguage must be a well-formed BCP-47 language tag",
|
||||
});
|
||||
if ((b.provider === undefined) !== (b.model === undefined)
|
||||
|| (b.provider !== undefined && (typeof b.provider !== "string" || typeof b.model !== "string" || !b.provider || !b.model))) {
|
||||
return reply.code(400).send({ error: "provider and model must be supplied together" });
|
||||
}
|
||||
const principal = getPrincipal(req);
|
||||
let s: Settings;
|
||||
try { s = await d.getSettings(principal); } catch { return storageFailure(reply); }
|
||||
@@ -379,6 +435,19 @@ export function sessionRoutes(
|
||||
workspaceId = resolved.revision.id;
|
||||
workspaceRevision = resolved.revision.commit;
|
||||
workspaceDescriptor = resolved.workspace;
|
||||
if (!await catalogTransportSupportsSessionRuntime(workspaceId)) {
|
||||
return reply.code(409).send({
|
||||
error: "This workspace transport is not available to runtime sessions.",
|
||||
code: "workspace_not_activatable",
|
||||
});
|
||||
}
|
||||
const fingerprint = await currentInputFingerprint(runner, workspaceConfigPath);
|
||||
if (!await preprocessingIsCurrent(workspaceId, fingerprint)) {
|
||||
return reply.code(409).send({
|
||||
error: "Run workspace preprocessing before starting the core.",
|
||||
code: "preprocessing_required",
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
return reply.code(409).send({
|
||||
error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE,
|
||||
@@ -387,11 +456,9 @@ export function sessionRoutes(
|
||||
}
|
||||
}
|
||||
const requestedCanonical = b.provider && b.model ? `${b.provider}/${b.model}` : undefined;
|
||||
let selectedCanonical = requestedCanonical ?? d.modelCatalog.defaultSession;
|
||||
let modelWarning: string | undefined;
|
||||
if (selectedCanonical && d.modelCatalog.defaultSession && !d.modelCatalog.hasSession(selectedCanonical)) {
|
||||
selectedCanonical = d.modelCatalog.defaultSession;
|
||||
modelWarning = `Configured model ${requestedCanonical ?? "selection"} is unavailable; using ${selectedCanonical}.`;
|
||||
const selectedCanonical = requestedCanonical ?? d.modelCatalog.defaultInteraction;
|
||||
if (selectedCanonical && d.modelCatalog.defaultInteraction && !d.modelCatalog.hasSession(selectedCanonical)) {
|
||||
return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
|
||||
}
|
||||
const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : undefined;
|
||||
const provider = selected?.provider ?? b.provider;
|
||||
@@ -442,11 +509,15 @@ export function sessionRoutes(
|
||||
// registry snapshot. The legacy fallback stays available for sessions created before
|
||||
// the browser-local preference migration.
|
||||
let id: string;
|
||||
let sessionLanguage = language;
|
||||
try {
|
||||
({ id } = await runner.sessionNew({
|
||||
const created = await runner.sessionNew({
|
||||
question: b.question, name: b.name, workspaceConfigPath,
|
||||
workspaceId, workspaceRevision, provider, model, thinking,
|
||||
}));
|
||||
interactionLanguage: language,
|
||||
});
|
||||
id = created.id;
|
||||
sessionLanguage = created.interaction_language ?? language;
|
||||
manifestPersisted = true;
|
||||
if (revisionLease) {
|
||||
await revisionLease.markPersisted().catch((error: unknown) => {
|
||||
@@ -459,6 +530,7 @@ export function sessionRoutes(
|
||||
} catch { return storageFailure(reply); }
|
||||
const options = {
|
||||
provider, model, thinking,
|
||||
interactionLanguage: sessionLanguage,
|
||||
author: principal.displayName ?? principal.subject,
|
||||
principal,
|
||||
question: b.question,
|
||||
@@ -495,7 +567,7 @@ export function sessionRoutes(
|
||||
),
|
||||
() => d.mgr.start(id, rt, runtimeOptions),
|
||||
);
|
||||
return { id, ...(modelWarning ? { warning: modelWarning } : {}) };
|
||||
return { id };
|
||||
} finally {
|
||||
if (revisionLease && !manifestPersisted) {
|
||||
await revisionLease.abort().catch((error: unknown) => {
|
||||
@@ -521,8 +593,8 @@ export function sessionRoutes(
|
||||
? { ...principal, isAdmin: false }
|
||||
: ownershipPrincipal(principal, "session.read_all");
|
||||
const runner = runnerFor(scopedPrincipal);
|
||||
const revisions = await sessionRevisions();
|
||||
const lists = await Promise.all(revisions
|
||||
const revisionScan = await sessionRevisions();
|
||||
const lists = await Promise.all(revisionScan.revisions
|
||||
.map((revision) => runner.sessionList(revision.snapshotPath) as Promise<SessionRow[]>));
|
||||
const sessions = new Map<string, SessionRow>();
|
||||
for (const row of lists.flat()) {
|
||||
@@ -532,7 +604,8 @@ export function sessionRoutes(
|
||||
// Only an administrator-visible complete list (or the single local principal) is safe
|
||||
// input for retention. A remote per-user view can never discard another principal's pin.
|
||||
const reconcileSnapshotRetention = (d.workspaceRegistry as Partial<WorkspaceRegistry>).reconcileSnapshotRetention;
|
||||
const hasCompleteRetentionView = (scope === "all" || principal.issuer === "local")
|
||||
const hasCompleteRetentionView = revisionScan.retainedComplete
|
||||
&& (scope === "all" || principal.issuer === "local")
|
||||
&& hasPermission(principal, "session.read_all");
|
||||
if (hasCompleteRetentionView && typeof reconcileSnapshotRetention === "function") {
|
||||
const retained = [...new Set(list
|
||||
@@ -566,6 +639,23 @@ export function sessionRoutes(
|
||||
} catch (error) { return lifecycleFailure(reply, error); }
|
||||
const rt = d.mgr.get(id);
|
||||
if (!rt) return reply.code(404).send({ error: "sessione non attiva" });
|
||||
const pending = rt.bridge.pendingWidget?.() as any;
|
||||
const response = (req.body as any)?.ui_response;
|
||||
if (pending?.widget === "archive-repair" && response?.id === pending.id && !response.control) {
|
||||
const choices = response.choices;
|
||||
if (!Array.isArray(choices) || choices.length !== 1)
|
||||
return reply.code(400).send({ error: "Select one archive repair choice" });
|
||||
if (choices[0] !== "continue" && choices[0] !== "reject") {
|
||||
const option = pending.repair?.options?.find((item: any) => item.id === choices[0]);
|
||||
if (!option || !["memory", "evidence"].includes(option.archive))
|
||||
return reply.code(400).send({ error: "Unknown archive repair choice" });
|
||||
const permission = option.archive === "memory" ? "memory.manage" : "evidence.manage";
|
||||
if (!principal.isAdmin || !hasPermission(principal, permission))
|
||||
return reply.code(403).send({ error: "Archive corrections require an administrator" });
|
||||
if (rt.ownerKey !== `${principal.issuer}\0${principal.subject}`)
|
||||
return reply.code(409).send({ error: "Resume the session with your account before correcting an archive" });
|
||||
}
|
||||
}
|
||||
if (!rt.bridge.respond((req.body as any).ui_response)) {
|
||||
return reply.code(409).send({ error: "risposta non corrispondente al gate in attesa" });
|
||||
}
|
||||
@@ -585,6 +675,13 @@ export function sessionRoutes(
|
||||
app.post("/sessions/:id/resume", async (req, reply) => {
|
||||
const id = (req.params as any).id;
|
||||
const principal = getPrincipal(req);
|
||||
if (Object.hasOwn(req.body ?? {}, "interactionLanguage")
|
||||
|| Object.hasOwn(req.body ?? {}, "interaction_language")) {
|
||||
return reply.code(400).send({
|
||||
code: "interaction_language_pinned",
|
||||
error: "Resume uses the session's persisted interaction language; overrides are not accepted",
|
||||
});
|
||||
}
|
||||
return withSessionLifecycle(id, async () => {
|
||||
let settings: Settings;
|
||||
let located: LocatedSession | undefined;
|
||||
@@ -602,11 +699,19 @@ export function sessionRoutes(
|
||||
const saved = manifest as {
|
||||
provider?: string; model?: string; thinking?: string;
|
||||
workspace_id?: string; workspace_revision?: string;
|
||||
interaction_language?: string | null;
|
||||
};
|
||||
const savedCanonical = saved.provider && saved.model ? `${saved.provider}/${saved.model}` : "";
|
||||
if (d.modelCatalog.defaultSession && (!savedCanonical || !d.modelCatalog.hasSession(savedCanonical))) {
|
||||
const requested = (req.body ?? {}) as { provider?: string; model?: string; thinking?: string };
|
||||
if ((requested.provider === undefined) !== (requested.model === undefined)
|
||||
|| (requested.provider !== undefined && (typeof requested.provider !== "string" || typeof requested.model !== "string" || !requested.provider || !requested.model))) {
|
||||
return reply.code(400).send({ error: "provider and model must be supplied together" });
|
||||
}
|
||||
const selectedCanonical = requested.provider && requested.model
|
||||
? `${requested.provider}/${requested.model}` : d.modelCatalog.defaultInteraction;
|
||||
if (d.modelCatalog.defaultInteraction && (!selectedCanonical || !d.modelCatalog.hasSession(selectedCanonical))) {
|
||||
return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
|
||||
}
|
||||
const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : saved;
|
||||
let workspaceConfigPath: string;
|
||||
let workspaceDescriptor: WorkspaceDescriptor | undefined;
|
||||
try {
|
||||
@@ -615,14 +720,37 @@ export function sessionRoutes(
|
||||
workspaceDescriptor = resolved.workspace;
|
||||
}
|
||||
catch { return unavailableWorkspaceReply(reply); }
|
||||
if (d.catalogRepository && saved.workspace_id
|
||||
&& !await catalogTransportSupportsSessionRuntime(saved.workspace_id)) {
|
||||
return reply.code(409).send({
|
||||
error: "This workspace transport is not available to runtime sessions.",
|
||||
code: "workspace_not_activatable",
|
||||
});
|
||||
}
|
||||
const fingerprint = d.catalogRepository
|
||||
? await currentInputFingerprint(runner, workspaceConfigPath)
|
||||
: undefined;
|
||||
if (d.catalogRepository
|
||||
&& (!saved.workspace_id || !await preprocessingIsCurrent(saved.workspace_id, fingerprint))) {
|
||||
return reply.code(409).send({
|
||||
error: "Run workspace preprocessing before starting the core.",
|
||||
code: "preprocessing_required",
|
||||
});
|
||||
}
|
||||
try { settings = await d.getSettings(principal); } catch { return storageFailure(reply); }
|
||||
let language = saved.interaction_language;
|
||||
if (language == null) {
|
||||
try {
|
||||
language = (await runner.ensureInteractionLanguage(id, workspaceConfigPath)).interaction_language;
|
||||
} catch { return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE }); }
|
||||
}
|
||||
// This check belongs inside the per-session lock: a preceding cold Resume may have
|
||||
// installed a running runtime while this request was waiting.
|
||||
const existing = d.mgr.get(id);
|
||||
if (existing) {
|
||||
const state = existing.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
return reply.code(200).send({ id, alreadyActive: true, workspaceId: saved.workspace_id });
|
||||
}
|
||||
}
|
||||
const ensure = await d.readiness.ensure(
|
||||
@@ -633,8 +761,9 @@ export function sessionRoutes(
|
||||
...(ensure.code ? { code: ensure.code } : {}),
|
||||
});
|
||||
const options = {
|
||||
provider: saved?.provider,
|
||||
model: saved?.model,
|
||||
provider: selected.provider,
|
||||
interactionLanguage: language,
|
||||
model: selected.model,
|
||||
thinking: saved?.thinking ?? settings.thinking,
|
||||
author: principal.displayName ?? principal.subject,
|
||||
principal,
|
||||
@@ -656,7 +785,7 @@ export function sessionRoutes(
|
||||
if (current) {
|
||||
const state = current.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
return reply.code(200).send({ id, alreadyActive: true, workspaceId: saved.workspace_id });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -697,7 +826,7 @@ export function sessionRoutes(
|
||||
d.mgr.configure(rt, runtimeOptions), null,
|
||||
() => d.mgr.start(id, rt, runtimeOptions),
|
||||
);
|
||||
return reply.code(200).send({ id, alreadyActive: false });
|
||||
return reply.code(200).send({ id, alreadyActive: false, workspaceId: saved.workspace_id });
|
||||
});
|
||||
});
|
||||
app.post("/sessions/:id/close", async (req, reply) => {
|
||||
|
||||
@@ -16,8 +16,8 @@ export function effectiveSettings(
|
||||
modelCatalog?: RuntimeModelCatalog,
|
||||
): Settings {
|
||||
const workspaces = listWorkspaces(cfg.harnessDir);
|
||||
const selected = modelCatalog?.defaultSession
|
||||
? splitCanonicalModelId(modelCatalog.defaultSession)
|
||||
const selected = modelCatalog?.defaultInteraction
|
||||
? splitCanonicalModelId(modelCatalog.defaultInteraction)
|
||||
: undefined;
|
||||
return {
|
||||
workspace: stored.workspace ?? workspaces[0]?.name,
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import {
|
||||
CatalogUnavailableError,
|
||||
type CatalogRepository,
|
||||
type WorkspaceDatabase,
|
||||
} from "../catalog/types.js";
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
|
||||
import type { PreprocessingJobState } from "../workspaces/preprocessing-state.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
|
||||
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
|
||||
const ACTIVE_SYNC_STATES = new Set(["queued", "running", "awaiting_confirmation", "applying"]);
|
||||
|
||||
export type WorkspacePreprocessingUiState =
|
||||
| "ready"
|
||||
| "required"
|
||||
| "running"
|
||||
| "blocked"
|
||||
| "failed";
|
||||
|
||||
export interface WorkspacePreprocessingStatus {
|
||||
schemaVersion: 1;
|
||||
workspaceId: string;
|
||||
state: WorkspacePreprocessingUiState;
|
||||
actionable: boolean;
|
||||
clearable: boolean;
|
||||
detail: string;
|
||||
reason?: string;
|
||||
nextStep?: string;
|
||||
metadataRevision?: number;
|
||||
preprocessedMetadataRevision?: number;
|
||||
startedAt?: string;
|
||||
finishedAt?: string;
|
||||
progress?: {
|
||||
stage: "catalog_snapshot" | "schema_index" | "evidence" | "finalizing";
|
||||
step: number;
|
||||
totalSteps: 4;
|
||||
};
|
||||
lastFailure?: {
|
||||
stage: string;
|
||||
errorCode: string;
|
||||
finishedAt: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface WorkspacePreprocessingRouteDeps {
|
||||
repository: CatalogRepository;
|
||||
registry: WorkspaceRegistry;
|
||||
service: Pick<WorkspacePreprocessingService, "run" | "clear">;
|
||||
inputFingerprint?: Pick<ThtRunner, "workspaceInputFingerprint">;
|
||||
readLatestJob?: (workspaceId: string) => PreprocessingJobState | undefined;
|
||||
}
|
||||
|
||||
const PROGRESS_DETAILS = {
|
||||
catalog_snapshot: "Preparing the PostgreSQL Catalog snapshot.",
|
||||
schema_index: "Building schema vectors and LSH indexes.",
|
||||
evidence: "Indexing Evidence.",
|
||||
finalizing: "Publishing the completed preprocessing state.",
|
||||
} as const;
|
||||
|
||||
function runningProgress(
|
||||
workspaceId: string,
|
||||
deps: WorkspacePreprocessingRouteDeps,
|
||||
): NonNullable<WorkspacePreprocessingStatus["progress"]> {
|
||||
let job: PreprocessingJobState | undefined;
|
||||
try {
|
||||
job = deps.readLatestJob?.(workspaceId);
|
||||
} catch {
|
||||
// Progress is supplemental. A damaged or temporarily unavailable checkpoint must not hide
|
||||
// the authoritative running state held by PostgreSQL.
|
||||
}
|
||||
const completed = new Set(job?.status === "active" ? job.completedStages : []);
|
||||
if (completed.has("evidence")) return { stage: "finalizing", step: 4, totalSteps: 4 };
|
||||
if (completed.has("schema_index")) return { stage: "evidence", step: 3, totalSteps: 4 };
|
||||
if (completed.has("catalog_snapshot")) return { stage: "schema_index", step: 2, totalSteps: 4 };
|
||||
return { stage: "catalog_snapshot", step: 1, totalSteps: 4 };
|
||||
}
|
||||
|
||||
function base(
|
||||
workspaceId: string,
|
||||
state: WorkspacePreprocessingUiState,
|
||||
detail: string,
|
||||
database?: WorkspaceDatabase,
|
||||
): WorkspacePreprocessingStatus {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
workspaceId,
|
||||
state,
|
||||
actionable: state === "ready" || state === "required" || state === "failed",
|
||||
clearable: Boolean(
|
||||
database
|
||||
&& state !== "running"
|
||||
&& database.preprocessingErrorCode !== "derived_data_cleared"
|
||||
),
|
||||
detail,
|
||||
...(database ? {
|
||||
metadataRevision: database.metadataContentRevision,
|
||||
...(database.preprocessedMetadataRevision === undefined
|
||||
? {}
|
||||
: { preprocessedMetadataRevision: database.preprocessedMetadataRevision }),
|
||||
...(database.preprocessingStartedAt ? { startedAt: database.preprocessingStartedAt } : {}),
|
||||
...(database.preprocessingFinishedAt ? { finishedAt: database.preprocessingFinishedAt } : {}),
|
||||
} : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function blocked(
|
||||
workspaceId: string,
|
||||
detail: string,
|
||||
reason: string,
|
||||
nextStep: string,
|
||||
database?: WorkspaceDatabase,
|
||||
): WorkspacePreprocessingStatus {
|
||||
return { ...base(workspaceId, "blocked", detail, database), reason, nextStep };
|
||||
}
|
||||
|
||||
function failureDiagnostic(errorCode: string): {
|
||||
stage: string;
|
||||
detail: string;
|
||||
reason: string;
|
||||
nextStep: string;
|
||||
} {
|
||||
switch (errorCode) {
|
||||
case "catalog_snapshot_failed":
|
||||
return {
|
||||
stage: "catalog_snapshot",
|
||||
detail: "The Catalog snapshot could not be prepared.",
|
||||
reason: "PostgreSQL Catalog metadata could not be read into a consistent preprocessing snapshot.",
|
||||
nextStep: "Check Catalog availability, then retry preprocessing.",
|
||||
};
|
||||
case "schema_index_failed":
|
||||
return {
|
||||
stage: "schema_index",
|
||||
detail: "The schema index could not be rebuilt.",
|
||||
reason: "The schema indexing worker stopped before the Catalog snapshot was published to Qdrant.",
|
||||
nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
|
||||
};
|
||||
case "evidence_preprocessing_failed":
|
||||
return {
|
||||
stage: "evidence",
|
||||
detail: "Evidence preprocessing did not complete.",
|
||||
reason: "The Evidence indexing worker stopped before it finished publishing the current workspace data.",
|
||||
nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
|
||||
};
|
||||
case "semantic_index_incompatible":
|
||||
return {
|
||||
stage: "semantic_preflight",
|
||||
detail: "The semantic index configuration is incompatible.",
|
||||
reason: "Qdrant rejected the collection configuration for the active embedding model.",
|
||||
nextStep: "Check embedding dimensions and Qdrant collection settings, then retry.",
|
||||
};
|
||||
case "egress_policy_refused":
|
||||
return {
|
||||
stage: "evidence",
|
||||
detail: "The Evidence source was refused by policy.",
|
||||
reason: "The configured Evidence endpoint is not allowed by the installation egress policy.",
|
||||
nextStep: "Correct the Evidence source or its allowlist configuration, then retry.",
|
||||
};
|
||||
case "workspace_not_activatable":
|
||||
return {
|
||||
stage: "runtime_preflight",
|
||||
detail: "The workspace runtime could not be prepared.",
|
||||
reason: "The active workspace or one of its required runtime bindings is not usable.",
|
||||
nextStep: "Check Workspace management and Database management, then retry.",
|
||||
};
|
||||
default:
|
||||
return {
|
||||
stage: "preprocessing",
|
||||
detail: "Preprocessing did not complete.",
|
||||
reason: "The preprocessing worker stopped before the current Catalog revision was published.",
|
||||
nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function readWorkspacePreprocessingStatus(
|
||||
workspaceId: string,
|
||||
deps: WorkspacePreprocessingRouteDeps,
|
||||
): Promise<WorkspacePreprocessingStatus> {
|
||||
const database = await deps.repository.getByWorkspace(workspaceId);
|
||||
if (!database) {
|
||||
return blocked(
|
||||
workspaceId,
|
||||
"Database configuration is required.",
|
||||
"This workspace has no database configuration in the PostgreSQL Catalog.",
|
||||
"Open Database management and configure the workspace database.",
|
||||
);
|
||||
}
|
||||
|
||||
if (database.preprocessingStatus === "running") {
|
||||
const progress = runningProgress(workspaceId, deps);
|
||||
return {
|
||||
...base(workspaceId, "running", PROGRESS_DETAILS[progress.stage], database),
|
||||
progress,
|
||||
};
|
||||
}
|
||||
if (database.binding.transport === "ssh_tunnel") {
|
||||
return blocked(
|
||||
workspaceId,
|
||||
"The database transport is not supported by the core runtime.",
|
||||
"The current database binding uses an SSH tunnel, which cannot be used by a ThothII session.",
|
||||
"Open Database management and select a supported runtime transport.",
|
||||
database,
|
||||
);
|
||||
}
|
||||
if (database.schemaSyncedVersion !== database.version) {
|
||||
return blocked(
|
||||
workspaceId,
|
||||
"Catalog synchronization is required.",
|
||||
`Database configuration v${database.version} is newer than the latest Catalog synchronization${database.schemaSyncedVersion === undefined ? "." : ` v${database.schemaSyncedVersion}.`}`,
|
||||
"Open Database management and run Synchronize schema.",
|
||||
database,
|
||||
);
|
||||
}
|
||||
|
||||
const [syncRuns, activeDescriptionRun, sensitivityRuns] = await Promise.all([
|
||||
deps.repository.listSyncRuns(database.id, 10),
|
||||
deps.repository.getActiveDescriptionGenerationRun(),
|
||||
deps.repository.listSensitivityAnalysisRuns(50),
|
||||
]);
|
||||
if (syncRuns.some((run) => ACTIVE_SYNC_STATES.has(run.state))) {
|
||||
return blocked(
|
||||
workspaceId,
|
||||
"Catalog synchronization is in progress.",
|
||||
"The Catalog is being synchronized and its metadata revision is not stable yet.",
|
||||
"Wait for schema synchronization to finish, then run preprocessing.",
|
||||
database,
|
||||
);
|
||||
}
|
||||
if (activeDescriptionRun?.databaseId === database.id
|
||||
&& ["queued", "running"].includes(activeDescriptionRun.status)) {
|
||||
return blocked(
|
||||
workspaceId,
|
||||
"Description generation is in progress.",
|
||||
"Catalog descriptions are still being generated for this database.",
|
||||
"Wait for description generation to finish, then run preprocessing.",
|
||||
database,
|
||||
);
|
||||
}
|
||||
if (sensitivityRuns.some((run) => run.databaseId === database.id && run.status === "running")) {
|
||||
return blocked(
|
||||
workspaceId,
|
||||
"Sensitivity analysis is in progress.",
|
||||
"Catalog sensitivity metadata is still being analyzed for this database.",
|
||||
"Wait for sensitivity analysis to finish, then run preprocessing.",
|
||||
database,
|
||||
);
|
||||
}
|
||||
|
||||
let inputFingerprint: string | undefined;
|
||||
try {
|
||||
const record = await deps.registry.read(workspaceId);
|
||||
if (deps.inputFingerprint) {
|
||||
inputFingerprint = await deps.inputFingerprint.workspaceInputFingerprint(
|
||||
record.revision.snapshotPath,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
return blocked(
|
||||
workspaceId,
|
||||
"The workspace runtime configuration is unavailable.",
|
||||
"The active workspace revision or one of its required database secrets could not be resolved.",
|
||||
"Check Workspace management and Database management before running preprocessing.",
|
||||
database,
|
||||
);
|
||||
}
|
||||
|
||||
const current = database.preprocessingStatus === "succeeded"
|
||||
&& database.preprocessedMetadataRevision === database.metadataContentRevision
|
||||
&& (inputFingerprint === undefined
|
||||
|| database.preprocessingInputFingerprint === inputFingerprint);
|
||||
if (current) {
|
||||
return base(
|
||||
workspaceId,
|
||||
"ready",
|
||||
`Catalog revision ${database.metadataContentRevision} is indexed.`,
|
||||
database,
|
||||
);
|
||||
}
|
||||
|
||||
if (database.preprocessingErrorCode === "derived_data_cleared") {
|
||||
return base(
|
||||
workspaceId,
|
||||
"required",
|
||||
"Reference vectors and LSH are empty. Memory is preserved.",
|
||||
database,
|
||||
);
|
||||
}
|
||||
|
||||
if (database.preprocessingStatus === "failed"
|
||||
&& database.preprocessingErrorCode
|
||||
&& database.preprocessingErrorCode !== "catalog_changed"
|
||||
&& database.preprocessingErrorCode !== "derived_data_cleared"
|
||||
&& database.preprocessingFinishedAt) {
|
||||
const diagnostic = failureDiagnostic(database.preprocessingErrorCode);
|
||||
return {
|
||||
...base(workspaceId, "failed", diagnostic.detail, database),
|
||||
reason: diagnostic.reason,
|
||||
nextStep: diagnostic.nextStep,
|
||||
lastFailure: {
|
||||
stage: diagnostic.stage,
|
||||
errorCode: database.preprocessingErrorCode,
|
||||
finishedAt: database.preprocessingFinishedAt,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return base(
|
||||
workspaceId,
|
||||
"required",
|
||||
`Catalog revision ${database.metadataContentRevision} is not indexed.`,
|
||||
database,
|
||||
);
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({
|
||||
code: "catalog_unavailable",
|
||||
message: "Database catalog is unavailable.",
|
||||
});
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({
|
||||
code: "preprocessing_request_invalid",
|
||||
message: "Preprocessing request is invalid.",
|
||||
});
|
||||
}
|
||||
return reply.code(500).send({
|
||||
code: "preprocessing_run_failed",
|
||||
message: "Preprocessing status could not be resolved.",
|
||||
});
|
||||
}
|
||||
|
||||
function workspaceIdFrom(request: FastifyRequest): string {
|
||||
return workspaceIdSchema.parse((request.params as { workspaceId?: unknown }).workspaceId);
|
||||
}
|
||||
|
||||
export function workspacePreprocessingRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: WorkspacePreprocessingRouteDeps,
|
||||
): void {
|
||||
app.get("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
|
||||
if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply;
|
||||
try {
|
||||
return await readWorkspacePreprocessingStatus(workspaceIdFrom(request), deps);
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
|
||||
if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
|
||||
try {
|
||||
const workspaceId = workspaceIdFrom(request);
|
||||
const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
|
||||
if (!before.actionable) {
|
||||
return reply.code(409).send({ ...before, code: "preprocessing_blocked" });
|
||||
}
|
||||
|
||||
const result = await deps.service.run({ workspaceId });
|
||||
const after = await readWorkspacePreprocessingStatus(workspaceId, deps);
|
||||
if (after.state === "ready") return after;
|
||||
if (after.state === "failed") {
|
||||
return reply.code(422).send({ ...after, code: "preprocessing_run_failed" });
|
||||
}
|
||||
if (after.state === "blocked" || after.state === "running") {
|
||||
return reply.code(409).send({ ...after, code: "preprocessing_blocked" });
|
||||
}
|
||||
return reply.code(500).send({
|
||||
code: "preprocessing_run_failed",
|
||||
message: `Preprocessing ended with ${result.code}.`,
|
||||
});
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.delete("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
|
||||
if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
|
||||
try {
|
||||
const workspaceId = workspaceIdFrom(request);
|
||||
const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
|
||||
if (!before.clearable) {
|
||||
return reply.code(409).send({ ...before, code: "preprocessing_clear_blocked" });
|
||||
}
|
||||
const result = await deps.service.clear({ workspaceId });
|
||||
if (result.status !== "succeeded") {
|
||||
return reply.code(result.code === "preprocessing_conflict" ? 409 : 500).send({
|
||||
code: result.code,
|
||||
message: "Preprocessing data could not be cleared.",
|
||||
});
|
||||
}
|
||||
return await readWorkspacePreprocessingStatus(workspaceId, deps);
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/** Validate/canonicalize the tag; available translation catalogs belong to the UI. */
|
||||
export function interactionLanguage(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
try {
|
||||
const [canonical] = Intl.getCanonicalLocales(value);
|
||||
return canonical;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
openSync, readFileSync, readSync, realpathSync, statSync, unlinkSync, writeFileSync,
|
||||
} from "node:fs";
|
||||
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { parseAllDocuments } from "yaml";
|
||||
import { parse, parseAllDocuments } from "yaml";
|
||||
import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js";
|
||||
import { secretValue, type SecretBundleConfig } from "../config/secret-bundle.js";
|
||||
import { renderWorkspaceRuntimeFromSnapshotPath } from "../workspaces/runtime-config-lease.js";
|
||||
@@ -22,6 +22,9 @@ import {
|
||||
} from "../workspaces/schema.js";
|
||||
import { reconcileCollection, type CollectionMode } from "../workspaces/qdrant-collection.js";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import type { CatalogRepository } from "../catalog/types.js";
|
||||
import { preprocessingInputFingerprint } from "../workspaces/effective-config.js";
|
||||
import { workspaceVectorCollections } from "../workspaces/vector-collections.js";
|
||||
|
||||
export interface ThtConfig extends SecretBundleConfig {
|
||||
thtBin: string;
|
||||
@@ -35,12 +38,14 @@ export interface ThtConfig extends SecretBundleConfig {
|
||||
/** "self_heal" for session admission (create missing collections/indexes), default "require_existing". */
|
||||
qdrantCollectionMode?: "self_heal" | "require_existing";
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
catalogRepository?: CatalogRepository;
|
||||
}
|
||||
|
||||
export interface RuntimeConfigLease {
|
||||
path: string;
|
||||
workspaceId: string;
|
||||
workspaceRevision: string;
|
||||
inputFingerprint: string;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
@@ -54,6 +59,7 @@ export interface SessionRow {
|
||||
author: string | null;
|
||||
workspace_id?: string | null;
|
||||
workspace_revision?: string | null;
|
||||
interaction_language?: string | null;
|
||||
archived?: boolean;
|
||||
}
|
||||
|
||||
@@ -79,6 +85,7 @@ export type SemanticReadinessCode = "workspace_not_activatable" | "semantic_inde
|
||||
export interface QdrantEnsureResult {
|
||||
ok: boolean;
|
||||
code?: SemanticReadinessCode;
|
||||
state?: "ready" | "created" | "repaired" | "upgraded";
|
||||
}
|
||||
|
||||
const REQUIRED_QDRANT_PAYLOAD_INDEXES = [
|
||||
@@ -213,37 +220,31 @@ export class ThtRunner {
|
||||
}
|
||||
|
||||
/** Render one immutable canonical registry revision into a backend-owned harness config. */
|
||||
acquireWorkspaceRuntime(
|
||||
workspaceConfigPath: string,
|
||||
effectiveRelationships?: string,
|
||||
): RuntimeConfigLease {
|
||||
const effectiveRelationshipsPath = effectiveRelationships === undefined
|
||||
async acquireWorkspaceRuntime(workspaceConfigPath: string): Promise<RuntimeConfigLease> {
|
||||
const identity = this.assertWorkspaceSnapshot(workspaceConfigPath);
|
||||
const catalogDatabase = this.cfg.catalogRepository === undefined
|
||||
? undefined
|
||||
: this.createRuntimeSnapshot(effectiveRelationships);
|
||||
let rendered: ReturnType<typeof renderWorkspaceRuntimeFromSnapshotPath>;
|
||||
try {
|
||||
rendered = renderWorkspaceRuntimeFromSnapshotPath({
|
||||
snapshotPath: workspaceConfigPath,
|
||||
harnessDir: this.cfg.harnessDir,
|
||||
configPath: this.cfg.configPath,
|
||||
dataRoot: this.cfg.dataRoot ?? (() => {
|
||||
throw new Error("registry workspace runtime requires an absolute data root");
|
||||
})(),
|
||||
secretRoots: this.cfg.secretRoots ?? [],
|
||||
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
|
||||
workspaceSecretStore: this.cfg.workspaceSecretStore,
|
||||
effectiveRelationshipsPath,
|
||||
});
|
||||
} catch (error) {
|
||||
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
|
||||
throw error;
|
||||
: await this.cfg.catalogRepository.getByWorkspace(identity.workspaceId);
|
||||
if (this.cfg.catalogRepository !== undefined && !catalogDatabase) {
|
||||
throw new Error("workspace database is not configured in the Catalog");
|
||||
}
|
||||
const rendered = renderWorkspaceRuntimeFromSnapshotPath({
|
||||
snapshotPath: workspaceConfigPath,
|
||||
harnessDir: this.cfg.harnessDir,
|
||||
configPath: this.cfg.configPath,
|
||||
dataRoot: this.cfg.dataRoot ?? (() => {
|
||||
throw new Error("registry workspace runtime requires an absolute data root");
|
||||
})(),
|
||||
secretRoots: this.cfg.secretRoots ?? [],
|
||||
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
|
||||
workspaceSecretStore: this.cfg.workspaceSecretStore,
|
||||
catalogDatabase,
|
||||
});
|
||||
let path: string;
|
||||
try {
|
||||
path = this.createRuntimeSnapshot(rendered.renderedConfig);
|
||||
} catch (error) {
|
||||
rendered.releaseSecrets();
|
||||
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
|
||||
throw error;
|
||||
}
|
||||
let released = false;
|
||||
@@ -251,16 +252,29 @@ export class ThtRunner {
|
||||
path,
|
||||
workspaceId: rendered.workspaceId,
|
||||
workspaceRevision: rendered.workspaceRevision,
|
||||
inputFingerprint: preprocessingInputFingerprint(
|
||||
rendered.workspaceId,
|
||||
rendered.workspaceRevision,
|
||||
parse(rendered.renderedConfig),
|
||||
),
|
||||
release: () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
this.cleanupRuntimeSnapshot(path);
|
||||
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
|
||||
rendered.releaseSecrets();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async workspaceInputFingerprint(workspaceConfigPath: string): Promise<string> {
|
||||
const lease = await this.acquireWorkspaceRuntime(workspaceConfigPath);
|
||||
try {
|
||||
return lease.inputFingerprint;
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
}
|
||||
|
||||
private runtimeSnapshotDirectory(): string {
|
||||
if (!this.cfg.runtimeSnapshotRoot) throw new Error("runtime snapshot root is not configured");
|
||||
if (!isAbsolute(this.cfg.runtimeSnapshotRoot)) throw new Error("runtime snapshot root must be absolute");
|
||||
@@ -392,13 +406,9 @@ export class ThtRunner {
|
||||
workspaceConfigPath && isAbsolute(workspaceConfigPath)
|
||||
&& !this.runtimeSnapshots.has(workspaceConfigPath)
|
||||
) {
|
||||
let runtime: RuntimeConfigLease;
|
||||
try {
|
||||
runtime = this.acquireWorkspaceRuntime(workspaceConfigPath);
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
return this.run(args, runtime.path, timeoutMs).finally(runtime.release);
|
||||
return this.acquireWorkspaceRuntime(workspaceConfigPath).then((runtime) => (
|
||||
this.run(args, runtime.path, timeoutMs).finally(runtime.release)
|
||||
));
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const env: NodeJS.ProcessEnv = { ...process.env };
|
||||
@@ -473,6 +483,7 @@ export class ThtRunner {
|
||||
|
||||
async sessionNew(o: {
|
||||
question: string;
|
||||
interactionLanguage?: string;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
thinking?: string;
|
||||
@@ -488,6 +499,7 @@ export class ThtRunner {
|
||||
["--provider", o.provider],
|
||||
["--model", o.model],
|
||||
["--thinking", o.thinking],
|
||||
["--interaction-language", o.interactionLanguage],
|
||||
["--name", o.name],
|
||||
["--workspace-id", o.workspaceId],
|
||||
["--workspace-revision", o.workspaceRevision],
|
||||
@@ -495,7 +507,7 @@ export class ThtRunner {
|
||||
if (v) a.push(f, v);
|
||||
}
|
||||
a.push("--json");
|
||||
return this.json<{ id: string }>(a, o.workspaceConfigPath ?? o.workspace);
|
||||
return this.json<{ id: string; interaction_language?: string }>(a, o.workspaceConfigPath ?? o.workspace);
|
||||
}
|
||||
|
||||
/** Build and persist the deterministic F1 retrieval pack for a new session. */
|
||||
@@ -524,6 +536,12 @@ export class ThtRunner {
|
||||
return this.json<unknown>(["session", "show", id, "--json"], workspace);
|
||||
}
|
||||
|
||||
ensureInteractionLanguage(id: string, workspace?: string) {
|
||||
return this.json<{ interaction_language: string }>(
|
||||
["session", "ensure-interaction-language", id, "--json"], workspace,
|
||||
);
|
||||
}
|
||||
|
||||
sqlPreview(id: string, p: { limit?: number; offset?: number }, workspace?: string) {
|
||||
// No positional FILE: the harness resolves sql_final.sql from the session
|
||||
// via _session_sql_file(cfg, session_id), which respects the workspace path.
|
||||
@@ -598,24 +616,26 @@ export class ThtRunner {
|
||||
} catch {
|
||||
return { ok: false, code: "workspace_not_activatable" };
|
||||
}
|
||||
const collection = {
|
||||
collection: descriptor.workspace.id,
|
||||
dimensions: this.cfg.semanticRuntime.internalEmbeddingDimensions,
|
||||
distance: "cosine" as const,
|
||||
};
|
||||
const collections = workspaceVectorCollections(descriptor.workspace.id);
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutSec) * 1000);
|
||||
try {
|
||||
const checked = await reconcileCollection({
|
||||
baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
|
||||
collection: collection.collection,
|
||||
dimensions: collection.dimensions,
|
||||
distance: collection.distance,
|
||||
mode,
|
||||
request: this.cfg.qdrantRequest ?? fetch,
|
||||
signal: controller.signal,
|
||||
});
|
||||
return checked;
|
||||
const checked = await Promise.all(Object.entries(collections).map(async ([purpose, collection]) =>
|
||||
await reconcileCollection({
|
||||
baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
|
||||
collection,
|
||||
dimensions: this.cfg.semanticRuntime.internalEmbeddingDimensions,
|
||||
distance: "cosine",
|
||||
mode: mode === "evidence_maintenance" && purpose === "memory" ? "self_heal" : mode,
|
||||
request: this.cfg.qdrantRequest ?? fetch,
|
||||
signal: controller.signal,
|
||||
})));
|
||||
if (!checked.every((result) => result.ok)) {
|
||||
return { ok: false, code: "semantic_index_incompatible" };
|
||||
}
|
||||
const state = (["upgraded", "repaired", "created", "ready"] as const)
|
||||
.find((candidate) => checked.some((result) => result.state === candidate));
|
||||
return { ok: true, ...(state ? { state } : {}) };
|
||||
} catch {
|
||||
return { ok: false, code: "workspace_not_activatable" };
|
||||
} finally {
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { closeSync, constants as fsConstants, openSync } from "node:fs";
|
||||
import { readdir, readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { parse } from "yaml";
|
||||
import { loadConfig } from "./config.js";
|
||||
import { loadConfig, type AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
import { WorkspaceRegistry } from "./workspaces/registry.js";
|
||||
import { publishDeterministicRuntimeConfigLease, renderActiveWorkspaceRuntime } from "./workspaces/runtime-config-lease.js";
|
||||
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
||||
import { WorkspacePreprocessingService, type WorkspaceOperationResult } from "./workspaces/preprocessing-service.js";
|
||||
import type { SessionInventoryRow } from "./workspaces/preprocessing-state.js";
|
||||
import { createCatalogRepository } from "./catalog/repository.js";
|
||||
import type { CatalogRepository } from "./catalog/types.js";
|
||||
|
||||
export interface WorkspaceMaintenanceIo {
|
||||
stdin: string;
|
||||
@@ -19,7 +18,7 @@ export interface WorkspaceMaintenanceIo {
|
||||
writeStderr(value: string): void;
|
||||
}
|
||||
|
||||
type Command = "inspect" | "preprocess-dwh" | "schema-suggest-fks" | "schema-check" | "schema-accept" | "index-schema" | "preprocess-evidence" | "preprocess-run" | "vector-inspect" | "vector-rebuild";
|
||||
type Command = "inspect" | "preprocess-run" | "preprocess-clear" | "evidence-consolidate" | "evidence-refresh" | "evidence-decide";
|
||||
|
||||
function failureResult(
|
||||
operation: string,
|
||||
@@ -64,15 +63,11 @@ function parseRequest(command: string, stdin: string): Record<string, unknown> {
|
||||
}
|
||||
const allowedByCommand: Record<string, readonly string[]> = {
|
||||
inspect: ["schemaVersion", "workspaceId"],
|
||||
"preprocess-dwh": ["schemaVersion", "workspaceId", "resumeRunId"],
|
||||
"schema-suggest-fks": ["schemaVersion", "workspaceId", "fromSql", "assume", "resumeRunId"],
|
||||
"schema-check": ["schemaVersion", "workspaceId", "annotationsYaml", "reviewedCandidatesDigest"],
|
||||
"schema-accept": ["schemaVersion", "workspaceId", "runId", "yes"],
|
||||
"index-schema": ["schemaVersion", "workspaceId", "resumeRunId"],
|
||||
"preprocess-evidence": ["schemaVersion", "workspaceId", "dryRun", "resumeRunId"],
|
||||
"preprocess-run": ["schemaVersion", "workspaceId", "resumeRunId"],
|
||||
"vector-inspect": ["schemaVersion", "workspaceId"],
|
||||
"vector-rebuild": ["schemaVersion", "workspaceId", "collection", "confirm", "destroy"],
|
||||
"preprocess-run": ["schemaVersion", "workspaceId"],
|
||||
"preprocess-clear": ["schemaVersion", "workspaceId"],
|
||||
"evidence-consolidate": ["schemaVersion", "workspaceId"],
|
||||
"evidence-refresh": ["schemaVersion", "workspaceId"],
|
||||
"evidence-decide": ["schemaVersion", "workspaceId", "sourceId", "revision", "decision"],
|
||||
};
|
||||
const allowed = allowedByCommand[command];
|
||||
if (!allowed) throw new Error("unknown command");
|
||||
@@ -89,57 +84,24 @@ function exitCodeFor(result: WorkspaceOperationResult): number {
|
||||
|
||||
async function dispatch(command: Command, service: WorkspacePreprocessingService, request: Record<string, unknown>): Promise<WorkspaceOperationResult> {
|
||||
switch (command) {
|
||||
case "evidence-refresh":
|
||||
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "refresh" });
|
||||
case "evidence-decide":
|
||||
if (typeof request.sourceId !== "string" || !/^[a-f0-9]{64}$/.test(request.sourceId)
|
||||
|| typeof request.revision !== "string" || !/^[a-f0-9]{64}$/.test(request.revision)
|
||||
|| (request.decision !== "keep" && request.decision !== "replace")) throw new Error("invalid request");
|
||||
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "decide",
|
||||
sourceId: request.sourceId, revision: request.revision, decision: request.decision });
|
||||
case "evidence-consolidate":
|
||||
return await service.consolidateEvidence({ workspaceId: request.workspaceId as string });
|
||||
case "inspect":
|
||||
return await service.inspect({ workspaceId: request.workspaceId as string });
|
||||
case "preprocess-dwh":
|
||||
return await service.preprocessDwh({
|
||||
workspaceId: request.workspaceId as string,
|
||||
resumeRunId: request.resumeRunId as string | undefined,
|
||||
});
|
||||
case "schema-suggest-fks":
|
||||
return await service.suggestFks({
|
||||
workspaceId: request.workspaceId as string,
|
||||
fromSql: request.fromSql as any,
|
||||
assume: request.assume as any,
|
||||
resumeRunId: request.resumeRunId as string | undefined,
|
||||
});
|
||||
case "schema-check":
|
||||
return await service.checkSchema({
|
||||
workspaceId: request.workspaceId as string,
|
||||
annotationsYaml: request.annotationsYaml as string | undefined,
|
||||
reviewedCandidatesDigest: request.reviewedCandidatesDigest as string | undefined,
|
||||
});
|
||||
case "schema-accept":
|
||||
return await service.acceptSchema({
|
||||
workspaceId: request.workspaceId as string,
|
||||
runId: request.runId as string,
|
||||
yes: request.yes === true,
|
||||
});
|
||||
case "index-schema":
|
||||
return await service.indexSchema({
|
||||
workspaceId: request.workspaceId as string,
|
||||
resumeRunId: request.resumeRunId as string | undefined,
|
||||
});
|
||||
case "preprocess-evidence":
|
||||
return await service.preprocessEvidence({
|
||||
workspaceId: request.workspaceId as string,
|
||||
dryRun: request.dryRun as boolean | undefined,
|
||||
resumeRunId: request.resumeRunId as string | undefined,
|
||||
});
|
||||
case "preprocess-run":
|
||||
return await service.run({
|
||||
workspaceId: request.workspaceId as string,
|
||||
resumeRunId: request.resumeRunId as string | undefined,
|
||||
});
|
||||
case "vector-inspect":
|
||||
return await service.vectorInspect({ workspaceId: request.workspaceId as string });
|
||||
case "vector-rebuild":
|
||||
return await service.vectorRebuild({
|
||||
workspaceId: request.workspaceId as string,
|
||||
collection: request.collection as string | undefined,
|
||||
confirm: request.confirm as string | undefined,
|
||||
destroy: request.destroy === true,
|
||||
});
|
||||
case "preprocess-clear":
|
||||
return await service.clear({ workspaceId: request.workspaceId as string });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,48 +140,32 @@ export async function runWorkspaceMaintenanceCli(
|
||||
|| message === "unexpected request field"
|
||||
|| message === "invalid workspace id";
|
||||
return command in {
|
||||
inspect: true, "preprocess-dwh": true, "schema-suggest-fks": true, "schema-check": true,
|
||||
"schema-accept": true,
|
||||
"index-schema": true, "preprocess-evidence": true, "preprocess-run": true,
|
||||
inspect: true, "preprocess-run": true, "preprocess-clear": true, "evidence-consolidate": true,
|
||||
"evidence-refresh": true, "evidence-decide": true,
|
||||
} ? (requestError ? 2 : 1) : 2;
|
||||
}
|
||||
}
|
||||
|
||||
async function readSessionInventory(dataRoot: string, workspaceId: string): Promise<readonly SessionInventoryRow[]> {
|
||||
const directory = join(dataRoot, "sessions", workspaceId, "sessions");
|
||||
try {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
const rows: SessionInventoryRow[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || entry.isSymbolicLink()) continue;
|
||||
try {
|
||||
const source = await readFile(join(directory, entry.name, "session_manifest.yaml"), "utf8");
|
||||
const manifest = parse(source) as Record<string, unknown>;
|
||||
rows.push({
|
||||
id: entry.name,
|
||||
status: typeof manifest.status === "string" ? manifest.status : "open",
|
||||
archived: manifest.archived === true,
|
||||
workspaceRevision: typeof manifest.workspace_revision === "string" ? manifest.workspace_revision : null,
|
||||
});
|
||||
} catch {
|
||||
// fail closed at mutation time by ignoring unreadable manifests from the resumable scan
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
export interface ProductionWorkspacePreprocessingDeps {
|
||||
config?: AppConfig;
|
||||
catalogRepository?: CatalogRepository;
|
||||
registry?: WorkspaceRegistry;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
runner?: ThtRunner;
|
||||
}
|
||||
|
||||
function createProductionService(): WorkspacePreprocessingService {
|
||||
const config = loadConfig(process.env, { surface: "workspace-maintenance" });
|
||||
const registry = new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const workspaceSecretStore = new WorkspaceSecretStore({
|
||||
export function createProductionWorkspacePreprocessingService(
|
||||
deps: ProductionWorkspacePreprocessingDeps = {},
|
||||
): WorkspacePreprocessingService {
|
||||
const config = deps.config ?? loadConfig(process.env, { surface: "workspace-maintenance" });
|
||||
const catalogRepository = deps.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const registry = deps.registry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const workspaceSecretStore = deps.workspaceSecretStore ?? new WorkspaceSecretStore({
|
||||
root: config.workspaceSecretStoreRoot,
|
||||
runtimeRoot: config.workspaceSecretRuntimeRoot,
|
||||
installationId: config.workspaceRegistry.installationId,
|
||||
});
|
||||
const runner = new ThtRunner({
|
||||
const runner = deps.runner ?? new ThtRunner({
|
||||
thtBin: config.thtBin,
|
||||
harnessDir: config.harnessDir,
|
||||
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
|
||||
@@ -232,16 +178,19 @@ function createProductionService(): WorkspacePreprocessingService {
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
});
|
||||
return new WorkspacePreprocessingService({
|
||||
dataRoot: config.dataRoot ?? "/data",
|
||||
embeddingDimensions: config.internalEmbeddingDimensions,
|
||||
httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "")
|
||||
.split(",").map((value) => value.trim()).filter((value) => value.length > 0),
|
||||
catalogRepository,
|
||||
acquireActiveRuntime: async (workspaceId) => {
|
||||
const catalogDatabase = await catalogRepository.getByWorkspace(workspaceId);
|
||||
if (!catalogDatabase) throw new Error("workspace database is not configured in the Catalog");
|
||||
const active = await renderActiveWorkspaceRuntime({
|
||||
workspaceId,
|
||||
registry,
|
||||
@@ -251,6 +200,7 @@ function createProductionService(): WorkspacePreprocessingService {
|
||||
dataRoot: config.dataRoot ?? "/data",
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
workspaceSecretStore,
|
||||
catalogDatabase,
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
@@ -258,38 +208,55 @@ function createProductionService(): WorkspacePreprocessingService {
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
});
|
||||
const configLease = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId,
|
||||
registry,
|
||||
registryConfig: config.workspaceRegistry,
|
||||
harnessDir: config.harnessDir,
|
||||
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
|
||||
dataRoot: config.dataRoot ?? "/data",
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
workspaceSecretStore,
|
||||
});
|
||||
return {
|
||||
workspace: active.workspace,
|
||||
workspaceId: active.workspaceId,
|
||||
workspaceRevision: active.workspaceRevision,
|
||||
descriptorBlob: active.descriptorBlob,
|
||||
catalogBlob: active.catalogBlob,
|
||||
configLease,
|
||||
};
|
||||
try {
|
||||
const configLease = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId,
|
||||
registry,
|
||||
registryConfig: config.workspaceRegistry,
|
||||
harnessDir: config.harnessDir,
|
||||
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
|
||||
dataRoot: config.dataRoot ?? "/data",
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
workspaceSecretStore,
|
||||
catalogDatabase,
|
||||
});
|
||||
return {
|
||||
workspace: active.workspace,
|
||||
workspaceId: active.workspaceId,
|
||||
workspaceRevision: active.workspaceRevision,
|
||||
descriptorBlob: active.descriptorBlob,
|
||||
catalogBlob: active.catalogBlob,
|
||||
configLease,
|
||||
};
|
||||
} finally {
|
||||
active.releaseSecrets();
|
||||
}
|
||||
},
|
||||
runChild: async ({ argv, configPath }) => {
|
||||
const configFd = openSync(configPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
||||
try {
|
||||
return await new Promise((resolve) => {
|
||||
const childEnvironment = { ...process.env };
|
||||
for (const name of [
|
||||
"THT_CATALOG_DATABASE_URL",
|
||||
"THT_CATALOG_DB_HOST",
|
||||
"THT_CATALOG_DB_PORT",
|
||||
"THT_CATALOG_DB_NAME",
|
||||
"THT_CATALOG_RUNTIME_USER",
|
||||
"THT_CATALOG_RUNTIME_PASSWORD_FILE",
|
||||
"THT_CATALOG_MIGRATOR_DATABASE_URL",
|
||||
"THT_CATALOG_MIGRATOR_USER",
|
||||
"THT_CATALOG_MIGRATOR_PASSWORD_FILE",
|
||||
]) delete childEnvironment[name];
|
||||
const child = spawn(config.thtBin, argv, {
|
||||
cwd: config.harnessDir,
|
||||
env: { ...process.env, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
|
||||
env: { ...childEnvironment, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
|
||||
stdio: ["ignore", "pipe", "pipe", configFd],
|
||||
});
|
||||
let stdout = "";
|
||||
@@ -303,9 +270,8 @@ function createProductionService(): WorkspacePreprocessingService {
|
||||
closeSync(configFd);
|
||||
}
|
||||
},
|
||||
listSessions: async (workspaceId) => await readSessionInventory(config.dataRoot ?? "/data", workspaceId),
|
||||
semanticPreflight: async (workspace) => {
|
||||
const result = await runner.qdrantEnsure(workspace, 30);
|
||||
const result = await runner.qdrantEnsure(workspace, 30, "self_heal");
|
||||
return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" };
|
||||
},
|
||||
evidencePreflight: async (workspace) => {
|
||||
@@ -330,7 +296,11 @@ if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).
|
||||
writeStdout: (value) => { stdout.push(value); },
|
||||
writeStderr: (value) => { stderr.push(value); },
|
||||
};
|
||||
const exitCode = await runWorkspaceMaintenanceCli(process.argv, createProductionService(), io);
|
||||
const exitCode = await runWorkspaceMaintenanceCli(
|
||||
process.argv,
|
||||
createProductionWorkspacePreprocessingService(),
|
||||
io,
|
||||
);
|
||||
process.stdout.write(stdout.join(""));
|
||||
if (stderr.length > 0) process.stderr.write(stderr.join("").slice(0, 64 * 1024));
|
||||
process.exit(exitCode);
|
||||
|
||||
@@ -90,6 +90,7 @@ export function resolveBinding(
|
||||
): ResolvedBinding {
|
||||
requireSupportedDescriptor(workspace);
|
||||
const descriptor = validateWorkspaceDescriptor(workspace);
|
||||
if (!descriptor.dwh) throw new Error("workspace database is not bound in the descriptor");
|
||||
const contract = buildInstallationContract(descriptor);
|
||||
const variables = contract.variables.filter((variable) => variable.role === role);
|
||||
const transportVariable = variables.find((variable) => variable.suffix === "TRANSPORT");
|
||||
@@ -165,7 +166,9 @@ export function resolveRuntimeBindings(
|
||||
const descriptor = validateWorkspaceDescriptor(workspace);
|
||||
|
||||
return {
|
||||
dwh: resolveBinding(descriptor, "DWH", env, secretRoots),
|
||||
dwh: descriptor.dwh
|
||||
? resolveBinding(descriptor, "DWH", env, secretRoots)
|
||||
: { transport: "postgres_direct", values: {}, missing: [] },
|
||||
evidence: resolveEvidenceBinding(descriptor, env, secretRoots),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -155,7 +155,9 @@ export function buildInstallationContract(workspace: WorkspaceDescriptor): Insta
|
||||
workspaceId: descriptor.workspace.id,
|
||||
namespace,
|
||||
variables: [
|
||||
...connectorVariables(namespace, descriptor.dwh.supported_transports),
|
||||
...(descriptor.dwh
|
||||
? connectorVariables(namespace, descriptor.dwh.supported_transports)
|
||||
: []),
|
||||
...evidenceVariables(namespace, descriptor),
|
||||
],
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
import type { WorkspaceErrorCode } from "./types.js";
|
||||
import type { SemanticRuntimeConfig } from "./runtime-renderer.js";
|
||||
import type { AuthDiagnostics } from "../auth/diagnostics.js";
|
||||
import { workspaceVectorCollections } from "./vector-collections.js";
|
||||
|
||||
export interface Diagnostic {
|
||||
level: "error" | "warning" | "info";
|
||||
@@ -447,6 +448,9 @@ async function diagnoseValidatedWorkspace(
|
||||
|
||||
let activatable = true;
|
||||
if (!skipDwh) {
|
||||
if (!descriptor.dwh) {
|
||||
return { activatable: false, diagnostics: [diagnosticError("binding_missing", "dwh")] };
|
||||
}
|
||||
const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs);
|
||||
const dwhValues = bindings.dwh.values;
|
||||
const dwhField = (suffix: string) => bindingName(descriptor, suffix);
|
||||
@@ -514,20 +518,18 @@ async function diagnoseValidatedWorkspace(
|
||||
}
|
||||
|
||||
try {
|
||||
const vector = await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
|
||||
baseUrl: semanticRuntime.internalQdrantUrl,
|
||||
collection: descriptor.workspace.id,
|
||||
timeoutMs,
|
||||
signal,
|
||||
}));
|
||||
const expected = {
|
||||
collection: descriptor.workspace.id,
|
||||
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
||||
distance: "cosine",
|
||||
};
|
||||
if (vector.collection !== expected.collection
|
||||
|| vector.dimensions !== expected.dimensions
|
||||
|| vector.distance !== expected.distance) {
|
||||
const expectedCollections = Object.values(workspaceVectorCollections(descriptor.workspace.id));
|
||||
const vectors = await Promise.all(expectedCollections.map(async (collection) =>
|
||||
await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
|
||||
baseUrl: semanticRuntime.internalQdrantUrl,
|
||||
collection,
|
||||
timeoutMs,
|
||||
signal,
|
||||
}))));
|
||||
if (vectors.some((vector, index) =>
|
||||
vector.collection !== expectedCollections[index]
|
||||
|| vector.dimensions !== semanticRuntime.internalEmbeddingDimensions
|
||||
|| vector.distance !== "cosine")) {
|
||||
diagnostics.push(diagnosticError("semantic_index_incompatible"));
|
||||
activatable = false;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
|
||||
import { normalize } from "node:path";
|
||||
|
||||
export interface CanonicalEffectiveConfig {
|
||||
schemaVersion: 2;
|
||||
schemaVersion: 3;
|
||||
dwh: CanonicalDwhConfig;
|
||||
vector: CanonicalVectorConfig;
|
||||
embedding: CanonicalEmbeddingConfig;
|
||||
@@ -21,7 +21,10 @@ export interface CanonicalDwhConfig {
|
||||
}
|
||||
|
||||
export interface CanonicalVectorConfig {
|
||||
collection: string;
|
||||
collections: {
|
||||
reference: string;
|
||||
memory: string;
|
||||
};
|
||||
dimensions: number;
|
||||
distance: string;
|
||||
}
|
||||
@@ -120,7 +123,10 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
|
||||
if (!vector) {
|
||||
throw new TypeError("effective config is missing vector resources");
|
||||
}
|
||||
const collection = requireString(vector, "collection");
|
||||
const collections = asRecord(vector.collections);
|
||||
if (!collections) {
|
||||
throw new TypeError("effective config is missing vector collections");
|
||||
}
|
||||
const semanticIndex = asRecord(rendered.semantic_index);
|
||||
const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined;
|
||||
const dimensions = vectorStore
|
||||
@@ -129,7 +135,14 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
|
||||
const distance = vectorStore
|
||||
? requireString(vectorStore, "distance")
|
||||
: (optionalString(vector, "distance") ?? "cosine");
|
||||
return { collection, dimensions, distance };
|
||||
return {
|
||||
collections: {
|
||||
reference: requireString(collections, "reference"),
|
||||
memory: requireString(collections, "memory"),
|
||||
},
|
||||
dimensions,
|
||||
distance,
|
||||
};
|
||||
}
|
||||
|
||||
function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbeddingConfig {
|
||||
@@ -169,7 +182,7 @@ export function buildCanonicalEffectiveConfig(renderedConfig: unknown): Canonica
|
||||
throw new TypeError("effective config requires a rendered configuration object");
|
||||
}
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
schemaVersion: 3,
|
||||
dwh: buildDwhConfig(rendered),
|
||||
vector: buildVectorConfig(rendered),
|
||||
embedding: buildEmbeddingConfig(rendered),
|
||||
@@ -220,3 +233,20 @@ export function configFingerprint(renderedConfig: unknown): string {
|
||||
export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string {
|
||||
return sha256(effectiveConfigIdentity(workspaceId, renderedConfig));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fingerprint every non-Catalog input consumed by complete preprocessing. The immutable Git
|
||||
* revision covers the workspace descriptor and its revision-pinned Evidence tree; the effective
|
||||
* configuration identity covers the Catalog-derived DWH binding and semantic runtime contract.
|
||||
*/
|
||||
export function preprocessingInputFingerprint(
|
||||
workspaceId: string,
|
||||
workspaceRevision: string,
|
||||
renderedConfig: unknown,
|
||||
): string {
|
||||
return sha256(JSON.stringify({
|
||||
workspaceId,
|
||||
workspaceRevision,
|
||||
effectiveConfigIdentity: effectiveConfigIdentity(workspaceId, renderedConfig),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ export interface EvidencePreprocessingRequest {
|
||||
evidence: EvidenceConfig;
|
||||
job: EvidenceJobState;
|
||||
dryRun?: boolean;
|
||||
consolidate?: boolean;
|
||||
httpPrivateHostAllowlist?: readonly string[];
|
||||
}
|
||||
|
||||
@@ -56,7 +57,7 @@ function isPrivateHost(hostname: string): boolean {
|
||||
return hostname.endsWith(".internal");
|
||||
}
|
||||
|
||||
function evidencePolicy(
|
||||
export function evidencePolicy(
|
||||
evidence: EvidenceConfig,
|
||||
httpPrivateHostAllowlist?: readonly string[],
|
||||
): EvidencePreprocessingOutcome | undefined {
|
||||
@@ -95,13 +96,14 @@ function jobResult(job: EvidenceJobState): Pick<
|
||||
};
|
||||
}
|
||||
|
||||
async function runEvidenceStage(
|
||||
export async function runEvidenceStage(
|
||||
request: EvidencePreprocessingRequest,
|
||||
deps: EvidencePreprocessingDependencies,
|
||||
): Promise<EvidencePreprocessingOutcome> {
|
||||
const payload = await deps.runStage([
|
||||
"preprocess",
|
||||
"evidence",
|
||||
...(request.consolidate ? ["--consolidate"] : []),
|
||||
...(request.dryRun ? ["--dry-run"] : []),
|
||||
...(request.job.childRuns.evidence
|
||||
? ["--resume", request.job.childRuns.evidence]
|
||||
@@ -171,6 +173,14 @@ export async function continueEvidencePreprocessing(
|
||||
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
|
||||
if (policy) return { ...policy, ...jobResult(request.job) };
|
||||
if (!request.job.completedStages.includes("evidence")) {
|
||||
const preflight = await deps.evidencePreflight();
|
||||
if (!preflight.ok) {
|
||||
return {
|
||||
status: "failed",
|
||||
code: preflight.code,
|
||||
...jobResult(request.job),
|
||||
};
|
||||
}
|
||||
return await runEvidenceStage(request, deps);
|
||||
}
|
||||
return { status: "unchanged", code: "ok", ...jobResult(request.job) };
|
||||
|
||||
@@ -1,22 +1,21 @@
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { renameSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
continueEvidencePreprocessing,
|
||||
preprocessEvidence as runEvidencePreprocessing,
|
||||
evidencePolicy,
|
||||
runEvidenceStage,
|
||||
type EvidencePreprocessingDependencies,
|
||||
type EvidencePreprocessingOutcome,
|
||||
} from "./evidence/preprocessing.js";
|
||||
import type { WorkspaceDescriptor } from "./schema.js";
|
||||
import {
|
||||
PreprocessingStateStore,
|
||||
type FkReviewRecord,
|
||||
type PreprocessingJobState,
|
||||
type SessionInventoryRow,
|
||||
} from "./preprocessing-state.js";
|
||||
import type { DeterministicRuntimeConfigLease } from "./runtime-config-lease.js";
|
||||
import { readAnnotationsSync } from "./annotations-sync.js";
|
||||
import { reconcileCollection } from "./qdrant-collection.js";
|
||||
import { buildCatalogMetadataSnapshot } from "../catalog/metadata-snapshot.js";
|
||||
import type { CatalogRepository } from "../catalog/types.js";
|
||||
|
||||
export interface WorkspaceOperationResult {
|
||||
schemaVersion: 1;
|
||||
@@ -27,7 +26,7 @@ export interface WorkspaceOperationResult {
|
||||
| "preprocessing_resume_mismatch" | "manual_review_required"
|
||||
| "evidence_materialization_required" | "effective_config_mismatch"
|
||||
| "semantic_index_incompatible" | "annotation_invalid"
|
||||
| "egress_policy_refused";
|
||||
| "egress_policy_refused" | "catalog_not_ready" | "preprocessing_clear_failed";
|
||||
workspaceId: string;
|
||||
workspaceRevision: string;
|
||||
descriptorBlob: string;
|
||||
@@ -69,7 +68,6 @@ export interface WorkspacePreprocessingServiceDeps {
|
||||
dataRoot: string;
|
||||
acquireActiveRuntime(workspaceId: string): Promise<ActiveRuntime>;
|
||||
runChild(request: ChildProcessRequest): Promise<ChildProcessResult>;
|
||||
listSessions(workspaceId: string): Promise<readonly SessionInventoryRow[]>;
|
||||
semanticPreflight(workspace: WorkspaceDescriptor): Promise<
|
||||
{ ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
|
||||
>;
|
||||
@@ -77,7 +75,7 @@ export interface WorkspacePreprocessingServiceDeps {
|
||||
{ ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
|
||||
>;
|
||||
httpPrivateHostAllowlist?: readonly string[];
|
||||
embeddingDimensions?: number;
|
||||
catalogRepository?: CatalogRepository;
|
||||
}
|
||||
|
||||
interface RunScope {
|
||||
@@ -86,10 +84,6 @@ interface RunScope {
|
||||
job: PreprocessingJobState;
|
||||
}
|
||||
|
||||
function digest(value: string | Buffer): string {
|
||||
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
||||
}
|
||||
|
||||
function baseResult(
|
||||
runtime: ActiveRuntime,
|
||||
operation: string,
|
||||
@@ -116,41 +110,72 @@ function baseResult(
|
||||
export class WorkspacePreprocessingService {
|
||||
constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
|
||||
|
||||
|
||||
async vectorInspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
async evidenceSources(options: { workspaceId: string; action: "refresh" | "decide";
|
||||
sourceId?: string; revision?: string; decision?: "keep" | "replace"; actor?: string }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
const collection = runtime.workspace.workspace.id;
|
||||
const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" });
|
||||
if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] });
|
||||
const body = await res.json() as any;
|
||||
const info = body?.result;
|
||||
const vectors = info?.config?.params?.vectors;
|
||||
return baseResult(runtime, "vector inspect", "succeeded", "ok", {
|
||||
counts: { dimensions: vectors?.size ?? 0 },
|
||||
warnings: [`collection=${collection} distance=${vectors?.distance ?? "unknown"}`],
|
||||
});
|
||||
}
|
||||
|
||||
async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
const collection = runtime.workspace.workspace.id;
|
||||
if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) {
|
||||
return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] });
|
||||
try {
|
||||
if (options.action === "refresh") {
|
||||
const refused = evidencePolicy(runtime.workspace.evidence, this.deps.httpPrivateHostAllowlist);
|
||||
if (refused) return baseResult(runtime, "evidence refresh", "failed", refused.code);
|
||||
}
|
||||
const argv = ["evidence", "sources", options.action, "--json", "-c", "/dev/fd/3"];
|
||||
if (options.action === "decide") {
|
||||
if (!/^[a-f0-9]{64}$/.test(options.sourceId ?? "") || !/^[a-f0-9]{64}$/.test(options.revision ?? "")
|
||||
|| !["keep", "replace"].includes(options.decision ?? "")) throw new Error("Invalid source decision");
|
||||
const preflight = await this.deps.evidencePreflight(runtime.workspace);
|
||||
if (!preflight.ok) return baseResult(runtime, "evidence sources", "failed", preflight.code);
|
||||
argv.push("--source-id", options.sourceId!, "--revision", options.revision!, "--decision", options.decision!);
|
||||
}
|
||||
argv.push("--actor", options.actor ?? "installation operator");
|
||||
const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.exitCode !== 0 || payload.status !== "succeeded") throw new Error(
|
||||
typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence source operation failed");
|
||||
return baseResult(runtime, `evidence ${options.action}`, "succeeded", "ok", {
|
||||
counts: this.numberRecord(payload.counts),
|
||||
warnings: [options.action === "refresh" ? "Source comparisons are ready in Evidence management. Active content is unchanged."
|
||||
: "Source decision activated locally. Commit and push the Evidence tree manually."],
|
||||
});
|
||||
} catch (error) {
|
||||
return baseResult(runtime, `evidence ${options.action}`, "failed", "evidence_materialization_required", {
|
||||
warnings: [error instanceof Error ? error.message : "Evidence source operation failed"],
|
||||
});
|
||||
}
|
||||
const q = `${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`;
|
||||
const del = await fetch(q, { method: "DELETE" });
|
||||
if (!del.ok && del.status !== 404) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection delete failed"] });
|
||||
// Recreate the complete contract (dimensions + distance + the 8 required keyword indexes).
|
||||
const recreated = await reconcileCollection({
|
||||
baseUrl: runtime.configLease.semanticQdrantUrl,
|
||||
collection,
|
||||
dimensions: this.deps.embeddingDimensions ?? 1024,
|
||||
distance: "cosine",
|
||||
mode: "self_heal",
|
||||
});
|
||||
if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] });
|
||||
return baseResult(runtime, "vector rebuild", "succeeded", "ok", { warnings: [`recreated collection=${collection}`] });
|
||||
}
|
||||
|
||||
async consolidateEvidence(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
const preflight = await this.deps.evidencePreflight(runtime.workspace);
|
||||
if (!preflight.ok) return baseResult(runtime, "evidence consolidate", "failed", preflight.code);
|
||||
// Evidence has its own durable archive/corpus jobs. Do not alter Catalog readiness
|
||||
// or the full preprocessing job when publishing this one component.
|
||||
try {
|
||||
const outcome = await runEvidenceStage({ evidence: runtime.workspace.evidence,
|
||||
consolidate: true, job: { runId: randomBytes(16).toString("hex"), completedStages: [], childRuns: {} },
|
||||
}, {
|
||||
runStage: async argv => {
|
||||
const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.exitCode !== 0 || payload.status !== "succeeded") {
|
||||
return Promise.reject(new Error(typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence consolidation failed. Retry the command."));
|
||||
}
|
||||
return payload;
|
||||
},
|
||||
persistJob: () => undefined,
|
||||
evidencePreflight: async () => preflight,
|
||||
requireRunId: value => this.requireRunId(value),
|
||||
numberRecord: value => this.numberRecord(value),
|
||||
});
|
||||
return baseResult(runtime, "evidence consolidate", "succeeded", "ok", {
|
||||
...outcome, warnings: ["Evidence is active locally. Catalog and Schema readiness are unchanged. Commit and push the Evidence files manually."],
|
||||
});
|
||||
} catch (error) {
|
||||
return baseResult(runtime, "evidence consolidate", "failed", "evidence_materialization_required", {
|
||||
warnings: [error instanceof Error ? error.message : "Evidence consolidation failed. Retry the command."],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
try {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
@@ -175,227 +200,151 @@ export class WorkspacePreprocessingService {
|
||||
}
|
||||
}
|
||||
|
||||
async preprocessDwh(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
||||
const scope = await this.startRun(options.workspaceId, "preprocess dwh", options.resumeRunId);
|
||||
if (scope.job.completedStages.includes("dwh")) {
|
||||
return baseResult(scope.runtime, "preprocess dwh", "unchanged", "ok", {
|
||||
runId: scope.job.runId,
|
||||
childRuns: scope.job.childRuns,
|
||||
completedStages: [...scope.job.completedStages],
|
||||
});
|
||||
async run(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
if (!this.deps.catalogRepository) {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
return baseResult(runtime, "preprocess run", "failed", "catalog_not_ready");
|
||||
}
|
||||
const payload = await this.runJsonStage(scope.runtime, [
|
||||
"preprocess", "dwh", "--steps", "introspect,lsh",
|
||||
...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
]);
|
||||
const childRun = this.requireRunId(payload.run_id);
|
||||
scope.job.childRuns.dwh = childRun;
|
||||
if (!scope.job.completedStages.includes("dwh")) scope.job.completedStages.push("dwh");
|
||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
||||
return baseResult(scope.runtime, "preprocess dwh", "succeeded", "ok", {
|
||||
runId: scope.job.runId,
|
||||
childRuns: { ...scope.job.childRuns },
|
||||
completedStages: [...scope.job.completedStages],
|
||||
});
|
||||
return await this.runFromCatalog(options);
|
||||
}
|
||||
|
||||
async suggestFks(options: {
|
||||
workspaceId: string;
|
||||
fromSql?: ReadonlyArray<{ name: string; sql: string }>;
|
||||
assume?: readonly string[];
|
||||
resumeRunId?: string;
|
||||
}): Promise<WorkspaceOperationResult> {
|
||||
const scope = await this.startRun(options.workspaceId, "schema suggest-fks", options.resumeRunId);
|
||||
return await this.runSuggestStage(scope, options.fromSql ?? [], options.assume ?? []);
|
||||
}
|
||||
|
||||
async checkSchema(options: {
|
||||
workspaceId: string;
|
||||
annotationsYaml?: string;
|
||||
reviewedCandidatesDigest?: string;
|
||||
}): Promise<WorkspaceOperationResult> {
|
||||
async clear(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
const state = this.state(runtime.workspaceId);
|
||||
if ((options.annotationsYaml === undefined) !== (options.reviewedCandidatesDigest === undefined)) {
|
||||
return baseResult(runtime, "schema check", "failed", "annotation_invalid");
|
||||
const repository = this.deps.catalogRepository;
|
||||
if (!repository) return baseResult(runtime, "preprocess clear", "failed", "catalog_not_ready");
|
||||
const cleared = await repository.clearPreprocessing(runtime.workspaceId);
|
||||
if (cleared.kind !== "cleared") {
|
||||
return baseResult(
|
||||
runtime,
|
||||
"preprocess clear",
|
||||
"failed",
|
||||
cleared.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
|
||||
);
|
||||
}
|
||||
if (options.reviewedCandidatesDigest === undefined) {
|
||||
const payload = await this.runJsonStage(runtime, ["schema", "check", "--json", "-c", "/dev/fd/3"]);
|
||||
return baseResult(runtime, "schema check", Number(payload.orphan_count ?? 0) === 0 ? "succeeded" : "failed", Number(payload.orphan_count ?? 0) === 0 ? "ok" : "annotation_invalid");
|
||||
}
|
||||
const reviewedCandidatesDigest = options.reviewedCandidatesDigest;
|
||||
const runId = this.findRunIdByCandidateDigest(state, reviewedCandidatesDigest);
|
||||
if (!runId) return baseResult(runtime, "schema check", "failed", "annotation_invalid");
|
||||
const request = await this.withStagedInputs(runtime.workspaceId, [
|
||||
{ flag: "--annotations", name: "annotations.yaml", contents: options.annotationsYaml! },
|
||||
], async (argv) => await this.runJsonStage(runtime, [
|
||||
"schema", "check", ...argv,
|
||||
"--reviewed-candidates", reviewedCandidatesDigest,
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
]));
|
||||
if (request.reviewed_candidates_digest !== reviewedCandidatesDigest || typeof request.annotations_digest !== "string") {
|
||||
return baseResult(runtime, "schema check", "failed", "annotation_invalid", { runId });
|
||||
}
|
||||
// P5 supersedes the host-file FK review: schema check is read-only validation and never
|
||||
// records a review. Only `schema accept` records a human review for the curated Git blob.
|
||||
return baseResult(runtime, "schema check", "succeeded", "ok", { runId });
|
||||
}
|
||||
|
||||
async acceptSchema(options: { workspaceId: string; runId: string; yes?: boolean }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
const state = this.state(runtime.workspaceId);
|
||||
if (options.yes !== true) {
|
||||
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
|
||||
runId: options.runId,
|
||||
warnings: ["accept requires --yes"],
|
||||
});
|
||||
}
|
||||
if (!/^[0-9a-f]{32}$/.test(options.runId)) {
|
||||
return baseResult(runtime, "schema accept", "failed", "annotation_invalid");
|
||||
}
|
||||
const candidate = state.readFkCandidates(options.runId);
|
||||
if (candidate === undefined) {
|
||||
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
|
||||
runId: options.runId,
|
||||
warnings: ["candidate run is unavailable"],
|
||||
});
|
||||
}
|
||||
const synced = readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision);
|
||||
if (synced === undefined || synced.contents.toString("utf8").trim() === "") {
|
||||
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
|
||||
runId: options.runId,
|
||||
warnings: ["curated annotations are not synchronized"],
|
||||
});
|
||||
}
|
||||
// The harness parser validates the curated blob against the physical schema; the recorded
|
||||
// candidate digest must round-trip and the blob digest must match the synced destination.
|
||||
const payload = await this.runJsonStage(runtime, [
|
||||
"schema", "check", "--reviewed-candidates", candidate.digest, "--json", "-c", "/dev/fd/3",
|
||||
]);
|
||||
if (payload.annotations_digest !== synced.contentDigest
|
||||
|| payload.reviewed_candidates_digest !== candidate.digest
|
||||
|| Number(payload.orphan_count ?? 0) !== 0) {
|
||||
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { runId: options.runId });
|
||||
}
|
||||
const review = state.writeFkReview(options.runId, {
|
||||
reviewedCandidatesDigest: candidate.digest,
|
||||
annotationsDigest: synced.contentDigest,
|
||||
workspaceRevision: runtime.workspaceRevision,
|
||||
blobId: synced.blobId,
|
||||
const result = await this.deps.runChild({
|
||||
argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
|
||||
configPath: runtime.configLease.path,
|
||||
});
|
||||
const job = state.readJob(options.runId);
|
||||
job.reviewDigest = review.digest;
|
||||
if (!job.completedStages.includes("fk_review")) job.completedStages.push("fk_review");
|
||||
state.writeJob(job);
|
||||
return baseResult(runtime, "schema accept", "succeeded", "ok", {
|
||||
runId: options.runId,
|
||||
completedStages: [...job.completedStages],
|
||||
artifactIdentities: [
|
||||
{ kind: "fk_review", digest: review.digest },
|
||||
{ kind: "annotations", digest: synced.contentDigest },
|
||||
],
|
||||
if (result.exitCode !== 0) {
|
||||
return baseResult(runtime, "preprocess clear", "failed", "preprocessing_clear_failed");
|
||||
}
|
||||
const payload = JSON.parse(result.stdout) as Record<string, unknown>;
|
||||
return baseResult(runtime, "preprocess clear", "succeeded", "ok", {
|
||||
counts: this.numberRecord(payload.counts),
|
||||
});
|
||||
}
|
||||
|
||||
async indexSchema(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
||||
const scope = await this.startRun(options.workspaceId, "index-schema", options.resumeRunId);
|
||||
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
||||
if (!semantic.ok) return baseResult(scope.runtime, "index-schema", "failed", semantic.code, { runId: scope.job.runId });
|
||||
if (scope.job.completedStages.includes("schema_index")) {
|
||||
return baseResult(scope.runtime, "index-schema", "unchanged", "ok", {
|
||||
runId: scope.job.runId,
|
||||
completedStages: [...scope.job.completedStages],
|
||||
});
|
||||
private async runFromCatalog(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
const scope = await this.startRun(options.workspaceId);
|
||||
const repository = this.deps.catalogRepository!;
|
||||
const fingerprint = scope.runtime.configLease.inputFingerprint;
|
||||
const started = await repository.beginPreprocessing(scope.runtime.workspaceId, fingerprint);
|
||||
if (started.kind !== "started") {
|
||||
scope.job.status = "failed";
|
||||
scope.state.writeJob(scope.job);
|
||||
return baseResult(
|
||||
scope.runtime,
|
||||
"preprocess run",
|
||||
"failed",
|
||||
started.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
|
||||
{ warnings: [`catalog=${started.kind}`] },
|
||||
);
|
||||
}
|
||||
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
|
||||
const counts = this.numberRecord(payload.counts);
|
||||
scope.job.completedStages.push("schema_index");
|
||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
||||
return baseResult(scope.runtime, "index-schema", "succeeded", "ok", {
|
||||
runId: scope.job.runId,
|
||||
completedStages: [...scope.job.completedStages],
|
||||
counts,
|
||||
});
|
||||
}
|
||||
|
||||
async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
||||
const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId);
|
||||
const outcome = await runEvidencePreprocessing(
|
||||
{
|
||||
evidence: scope.runtime.workspace.evidence,
|
||||
job: scope.job,
|
||||
dryRun: options.dryRun,
|
||||
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
|
||||
},
|
||||
this.evidenceDependencies(scope),
|
||||
);
|
||||
return this.evidenceResult(scope, "preprocess evidence", outcome);
|
||||
}
|
||||
|
||||
async run(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
||||
const scope = await this.startRun(options.workspaceId, "preprocess run", options.resumeRunId);
|
||||
if (!scope.job.completedStages.includes("dwh")) {
|
||||
const payload = await this.runJsonStage(scope.runtime, [
|
||||
"preprocess", "dwh", "--steps", "introspect,lsh",
|
||||
...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
]);
|
||||
scope.job.childRuns.dwh = this.requireRunId(payload.run_id);
|
||||
scope.job.completedStages.push("dwh");
|
||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
||||
}
|
||||
if (!scope.job.completedStages.includes("fk_suggest")) {
|
||||
const suggest = await this.runSuggestStage(scope, [], []);
|
||||
if (suggest.code === "manual_review_required") return suggest;
|
||||
}
|
||||
const candidate = this.state(scope.runtime.workspaceId).readFkCandidates(scope.job.runId);
|
||||
if (candidate && !scope.job.completedStages.includes("fk_review")) {
|
||||
// P5: continuation requires a review accepted for this candidate whose accepted blob digest
|
||||
// equals the current revision's synced annotations. A revision change (or a missing curated
|
||||
// blob) therefore records a new review checkpoint instead of silently reusing the old one.
|
||||
const accepted = this.findAcceptedReviewForDigest(scope.runtime.workspaceId, candidate.digest);
|
||||
const currentDigest = this.currentAnnotationsDigest(scope.runtime);
|
||||
if (accepted === undefined || currentDigest === undefined || accepted.annotationsDigest !== currentDigest) {
|
||||
return baseResult(scope.runtime, "preprocess run", "blocked", "manual_review_required", {
|
||||
runId: scope.job.runId,
|
||||
childRuns: { ...scope.job.childRuns },
|
||||
completedStages: [...scope.job.completedStages],
|
||||
artifactIdentities: [{ kind: "fk_candidates", digest: candidate.digest }],
|
||||
});
|
||||
const revision = started.database.metadataContentRevision;
|
||||
let finished = false;
|
||||
let failureCode = "catalog_snapshot_failed";
|
||||
try {
|
||||
const snapshot = await buildCatalogMetadataSnapshot(
|
||||
repository,
|
||||
scope.runtime.workspaceId,
|
||||
revision,
|
||||
);
|
||||
const snapshotPath = this.publishCatalogSnapshot(
|
||||
scope.runtime.workspaceId,
|
||||
JSON.stringify(snapshot),
|
||||
);
|
||||
this.completeStages(scope, "catalog_snapshot");
|
||||
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
||||
if (!semantic.ok) {
|
||||
await repository.finishPreprocessing(
|
||||
scope.runtime.workspaceId,
|
||||
revision,
|
||||
fingerprint,
|
||||
{ status: "failed", errorCode: semantic.code },
|
||||
);
|
||||
scope.job.status = "failed";
|
||||
scope.state.writeJob(scope.job);
|
||||
finished = true;
|
||||
return baseResult(scope.runtime, "preprocess run", "failed", semantic.code);
|
||||
}
|
||||
scope.job.reviewDigest = accepted.reviewedCandidatesDigest;
|
||||
scope.job.completedStages.push("fk_review");
|
||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
||||
|
||||
failureCode = "schema_index_failed";
|
||||
const payload = await this.runJsonStage(scope.runtime, [
|
||||
"preprocess",
|
||||
"catalog",
|
||||
"--catalog-metadata",
|
||||
snapshotPath,
|
||||
"--json",
|
||||
"-c",
|
||||
"/dev/fd/3",
|
||||
]);
|
||||
this.completeStages(scope, "catalog_metadata", "lsh", "schema_index");
|
||||
failureCode = "evidence_preprocessing_failed";
|
||||
const outcome = await continueEvidencePreprocessing(
|
||||
{
|
||||
evidence: scope.runtime.workspace.evidence,
|
||||
job: scope.job,
|
||||
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
|
||||
priorCounts: this.numberRecord(payload.counts),
|
||||
},
|
||||
this.evidenceDependencies(scope),
|
||||
);
|
||||
if (!["succeeded", "unchanged"].includes(outcome.status)) {
|
||||
await repository.finishPreprocessing(
|
||||
scope.runtime.workspaceId,
|
||||
revision,
|
||||
fingerprint,
|
||||
{ status: "failed", errorCode: outcome.code },
|
||||
);
|
||||
scope.job.status = "failed";
|
||||
scope.state.writeJob(scope.job);
|
||||
finished = true;
|
||||
return this.evidenceResult(scope, outcome);
|
||||
}
|
||||
// Evidence has been published. The only remaining operation is the atomic Catalog commit.
|
||||
this.completeStages(scope, "evidence");
|
||||
const completed = await repository.finishPreprocessing(
|
||||
scope.runtime.workspaceId,
|
||||
revision,
|
||||
fingerprint,
|
||||
{ status: "succeeded" },
|
||||
);
|
||||
if (!completed) throw new Error("catalog preprocessing completion lost its lease");
|
||||
scope.job.status = "succeeded";
|
||||
scope.state.writeJob(scope.job);
|
||||
finished = true;
|
||||
return this.evidenceResult(scope, outcome);
|
||||
} catch (error) {
|
||||
if (!finished) {
|
||||
await repository.finishPreprocessing(
|
||||
scope.runtime.workspaceId,
|
||||
revision,
|
||||
fingerprint,
|
||||
{ status: "failed", errorCode: failureCode },
|
||||
);
|
||||
}
|
||||
scope.job.status = "failed";
|
||||
scope.state.writeJob(scope.job);
|
||||
throw error;
|
||||
}
|
||||
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
||||
if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId });
|
||||
let schemaCounts: Record<string, number> | undefined;
|
||||
if (!scope.job.completedStages.includes("schema_index")) {
|
||||
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
|
||||
scope.job.completedStages.push("schema_index");
|
||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
||||
schemaCounts = this.numberRecord(payload.counts);
|
||||
}
|
||||
const outcome = await continueEvidencePreprocessing(
|
||||
{
|
||||
evidence: scope.runtime.workspace.evidence,
|
||||
job: scope.job,
|
||||
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
|
||||
priorCounts: schemaCounts,
|
||||
},
|
||||
this.evidenceDependencies(scope),
|
||||
);
|
||||
return this.evidenceResult(scope, "preprocess run", outcome);
|
||||
}
|
||||
|
||||
private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise<RunScope> {
|
||||
private async startRun(workspaceId: string): Promise<RunScope> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(workspaceId);
|
||||
const state = this.state(runtime.workspaceId);
|
||||
await state.assertSessionInventoryCompatible(runtime.workspaceRevision, await this.deps.listSessions(runtime.workspaceId));
|
||||
const job = await state.beginJob({
|
||||
operation,
|
||||
runId: resumeRunId,
|
||||
operation: "preprocess run",
|
||||
workspaceRevision: runtime.workspaceRevision,
|
||||
descriptorBlob: runtime.descriptorBlob,
|
||||
catalogBlob: runtime.catalogBlob,
|
||||
@@ -411,6 +360,28 @@ export class WorkspacePreprocessingService {
|
||||
return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId });
|
||||
}
|
||||
|
||||
private completeStages(scope: RunScope, ...stages: string[]): void {
|
||||
for (const stage of stages) {
|
||||
if (!scope.job.completedStages.includes(stage)) scope.job.completedStages.push(stage);
|
||||
}
|
||||
scope.state.writeJob(scope.job);
|
||||
}
|
||||
|
||||
private publishCatalogSnapshot(workspaceId: string, contents: string): string {
|
||||
const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing");
|
||||
mkdirSync(root, { recursive: true, mode: 0o700 });
|
||||
const target = join(root, "catalog-metadata.json");
|
||||
const staging = join(root, `.catalog-metadata-${randomBytes(6).toString("hex")}.json`);
|
||||
try {
|
||||
writeFileSync(staging, contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
|
||||
renameSync(staging, target);
|
||||
return target;
|
||||
} catch (error) {
|
||||
rmSync(staging, { force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private evidenceDependencies(scope: RunScope): EvidencePreprocessingDependencies {
|
||||
return {
|
||||
runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
|
||||
@@ -423,50 +394,10 @@ export class WorkspacePreprocessingService {
|
||||
|
||||
private evidenceResult(
|
||||
scope: RunScope,
|
||||
operation: "preprocess evidence" | "preprocess run",
|
||||
outcome: EvidencePreprocessingOutcome,
|
||||
): WorkspaceOperationResult {
|
||||
const { status, code, ...extra } = outcome;
|
||||
return baseResult(scope.runtime, operation, status, code, extra);
|
||||
}
|
||||
|
||||
private async runSuggestStage(
|
||||
scope: RunScope,
|
||||
fromSql: ReadonlyArray<{ name: string; sql: string }>,
|
||||
assume: readonly string[],
|
||||
): Promise<WorkspaceOperationResult> {
|
||||
const payload = await this.withStagedInputs(scope.runtime.workspaceId, fromSql.map((entry) => ({
|
||||
flag: "--from-sql",
|
||||
name: entry.name,
|
||||
contents: entry.sql,
|
||||
})), async (stagedArgv) => await this.runJsonStage(scope.runtime, [
|
||||
"schema", "suggest-fks", ...stagedArgv,
|
||||
...assume.flatMap((value) => ["--assume", value]),
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
]));
|
||||
const candidateCount = Number(payload.candidate_count ?? 0);
|
||||
const candidateYaml = typeof payload.candidate_yaml === "string" ? payload.candidate_yaml : "";
|
||||
let artifactIdentities: Array<{ kind: string; digest: string }> | undefined;
|
||||
if (candidateCount > 0) {
|
||||
const persisted = this.state(scope.runtime.workspaceId).writeFkCandidates(scope.job.runId, candidateYaml);
|
||||
scope.job.candidateDigest = persisted.digest;
|
||||
artifactIdentities = [{ kind: "fk_candidates", digest: persisted.digest }];
|
||||
}
|
||||
if (!scope.job.completedStages.includes("fk_suggest")) scope.job.completedStages.push("fk_suggest");
|
||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
||||
const resultExtra = {
|
||||
runId: scope.job.runId,
|
||||
completedStages: [...scope.job.completedStages],
|
||||
...(artifactIdentities ? { artifactIdentities } : {}),
|
||||
...(candidateYaml.length > 0 ? { suggestedFksYaml: candidateYaml } : {}),
|
||||
};
|
||||
if (candidateCount > 0) {
|
||||
return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "blocked", "manual_review_required", {
|
||||
...resultExtra,
|
||||
childRuns: { ...scope.job.childRuns },
|
||||
});
|
||||
}
|
||||
return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "succeeded", "ok", resultExtra);
|
||||
return baseResult(scope.runtime, "preprocess run", status, code, extra);
|
||||
}
|
||||
|
||||
private async runJsonStage(runtime: ActiveRuntime, argv: string[]): Promise<Record<string, unknown>> {
|
||||
@@ -485,54 +416,5 @@ export class WorkspacePreprocessingService {
|
||||
return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([key, nested]) => [key, Number(nested)]));
|
||||
}
|
||||
|
||||
private async withStagedInputs<T>(
|
||||
workspaceId: string,
|
||||
inputs: ReadonlyArray<{ flag: string; name: string; contents: string }>,
|
||||
fn: (argv: string[]) => Promise<T>,
|
||||
): Promise<T> {
|
||||
if (inputs.length === 0) return await fn([]);
|
||||
const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing", `.stage-${randomBytes(6).toString("hex")}`);
|
||||
mkdirSync(root, { recursive: true, mode: 0o700 });
|
||||
const argv: string[] = [];
|
||||
const paths: string[] = [];
|
||||
try {
|
||||
for (const input of inputs) {
|
||||
const path = join(root, input.name);
|
||||
writeFileSync(path, input.contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
|
||||
paths.push(path);
|
||||
argv.push(input.flag, path);
|
||||
}
|
||||
return await fn(argv);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private currentAnnotationsDigest(runtime: ActiveRuntime): string | undefined {
|
||||
return readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision)?.contentDigest;
|
||||
}
|
||||
|
||||
private findAcceptedReviewForDigest(
|
||||
workspaceId: string,
|
||||
digestValue: string,
|
||||
): FkReviewRecord | undefined {
|
||||
const state = this.state(workspaceId);
|
||||
for (const entry of readdirSync(state.fkReviewsDirectory(), { withFileTypes: true })) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.json$/.test(entry.name)) continue;
|
||||
const runId = entry.name.slice(0, -".json".length);
|
||||
const review = state.readFkReview(runId);
|
||||
if (review && review.reviewedCandidatesDigest === digestValue) return review;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private findRunIdByCandidateDigest(state: PreprocessingStateStore, digestValue: string): string | undefined {
|
||||
for (const entry of readdirSync(state.fkCandidatesDirectory(), { withFileTypes: true })) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.yaml$/.test(entry.name)) continue;
|
||||
const runId = entry.name.slice(0, -".yaml".length);
|
||||
if (state.readFkCandidates(runId)?.digest === digestValue) return runId;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -198,6 +198,7 @@ export class PreprocessingStateStore {
|
||||
runtimeConfigDirectory(): string { return join(this.root, "runtime-config"); }
|
||||
runtimeConfigManifestDirectory(): string { return join(this.root, "runtime-config-manifests"); }
|
||||
jobsDirectory(): string { return join(this.root, "jobs"); }
|
||||
latestJobPath(): string { return join(this.root, "latest-job.json"); }
|
||||
fkCandidatesDirectory(): string { return join(this.root, "fk-candidates"); }
|
||||
fkReviewsDirectory(): string { return join(this.root, "fk-reviews"); }
|
||||
jobPath(runId: string): string { return join(this.jobsDirectory(), `${validateRunId(runId)}.json`); }
|
||||
@@ -289,7 +290,7 @@ export class PreprocessingStateStore {
|
||||
"Workspace preprocessing resume no longer matches the pinned revision",
|
||||
);
|
||||
}
|
||||
return existing;
|
||||
return this.writeJob(existing);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
if (error instanceof PreprocessingStateError) throw error;
|
||||
@@ -318,19 +319,30 @@ export class PreprocessingStateStore {
|
||||
childRuns: {},
|
||||
status: "active",
|
||||
};
|
||||
writeAtomicFile(path, `${JSON.stringify(job)}
|
||||
`, 0o600);
|
||||
return job;
|
||||
return this.writeJob(job);
|
||||
}
|
||||
|
||||
readJob(runId: string): PreprocessingJobState {
|
||||
return decodeJob(JSON.parse(readTrustedFile(this.jobPath(runId))));
|
||||
}
|
||||
|
||||
readLatestJob(): PreprocessingJobState | undefined {
|
||||
try {
|
||||
return decodeJob(JSON.parse(readTrustedFile(this.latestJobPath())));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
writeJob(job: PreprocessingJobState): PreprocessingJobState {
|
||||
this.ensureLayout();
|
||||
writeAtomicFile(this.jobPath(job.runId), `${JSON.stringify(job)}
|
||||
`, 0o600);
|
||||
const contents = `${JSON.stringify(job)}
|
||||
`;
|
||||
writeAtomicFile(this.jobPath(job.runId), contents, 0o600);
|
||||
// This fixed pointer is the sole status surface for operators. Per-run files remain an
|
||||
// internal resume mechanism and are never exposed as history.
|
||||
writeAtomicFile(this.latestJobPath(), contents, 0o600);
|
||||
return job;
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
canonicalEffectiveConfigJson,
|
||||
configFingerprint,
|
||||
effectiveConfigIdentity,
|
||||
inputFingerprint,
|
||||
preprocessingInputFingerprint,
|
||||
type CanonicalEffectiveConfig,
|
||||
} from "./effective-config.js";
|
||||
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
|
||||
@@ -41,6 +41,8 @@ import {
|
||||
} from "./runtime-renderer.js";
|
||||
import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js";
|
||||
import type { WorkspaceRegistryConfig } from "./types.js";
|
||||
import { resolveCatalogRuntimeBinding } from "../catalog/runtime-binding.js";
|
||||
import type { WorkspaceDatabase } from "../catalog/types.js";
|
||||
|
||||
export interface RuntimeConfigLease {
|
||||
path: string;
|
||||
@@ -246,8 +248,6 @@ function readSnapshotWorkspace(snapshotPath: string): {
|
||||
function runtimePaths(
|
||||
dataRoot: string,
|
||||
workspaceId: string,
|
||||
workspaceRevision?: string,
|
||||
effectiveRelationshipsPath?: string,
|
||||
): RuntimePaths {
|
||||
if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
|
||||
const root = join(dataRoot, "sessions", workspaceId);
|
||||
@@ -256,12 +256,7 @@ function runtimePaths(
|
||||
artifacts: join(root, "artifacts"),
|
||||
indexes: join(root, "indexes"),
|
||||
memory: join(root, "memory"),
|
||||
...(workspaceRevision === undefined
|
||||
? {}
|
||||
: { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }),
|
||||
...(effectiveRelationshipsPath === undefined
|
||||
? {}
|
||||
: { effective_relationships: effectiveRelationshipsPath }),
|
||||
catalog_metadata_snapshot: join(root, "preprocessing", "catalog-metadata.json"),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -309,19 +304,32 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
|
||||
dataRoot: string;
|
||||
secretRoots: readonly string[];
|
||||
semanticRuntime: SemanticRuntimeConfig;
|
||||
effectiveRelationshipsPath?: string;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
catalogDatabase?: WorkspaceDatabase;
|
||||
}): RenderedWorkspaceRuntime {
|
||||
const secretLease = options.workspaceSecretStore === undefined
|
||||
if (options.catalogDatabase && !options.workspaceSecretStore) {
|
||||
throw new Error("Catalog runtime binding requires the workspace secret store");
|
||||
}
|
||||
const catalogLease = options.catalogDatabase === undefined
|
||||
? undefined
|
||||
: resolveCatalogRuntimeBinding({
|
||||
workspace: options.workspace,
|
||||
database: options.catalogDatabase,
|
||||
environment: process.env,
|
||||
secretRoots: options.secretRoots,
|
||||
secretStore: options.workspaceSecretStore!,
|
||||
});
|
||||
const runtimeWorkspace = catalogLease?.workspace ?? options.workspace;
|
||||
const secretLease = catalogLease !== undefined || options.workspaceSecretStore === undefined
|
||||
? undefined
|
||||
: resolveRuntimeBindingsWithWorkspaceSecrets(
|
||||
options.workspace,
|
||||
runtimeWorkspace,
|
||||
process.env,
|
||||
options.secretRoots,
|
||||
options.workspaceSecretStore,
|
||||
);
|
||||
const bindings = secretLease?.bindings
|
||||
?? resolveRuntimeBindings(options.workspace, process.env, options.secretRoots);
|
||||
const bindings = catalogLease?.bindings ?? secretLease?.bindings
|
||||
?? resolveRuntimeBindings(runtimeWorkspace, process.env, options.secretRoots);
|
||||
const overlay = installationOverlay(options.harnessDir, options.configPath);
|
||||
const context: RuntimeRenderContext = {
|
||||
workspaceId: options.workspaceId,
|
||||
@@ -334,32 +342,26 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
|
||||
workspaceId: options.workspaceId,
|
||||
workspaceRevision: options.workspaceRevision,
|
||||
revisionContentRoot: options.revisionContentRoot,
|
||||
runtimePaths: runtimePaths(
|
||||
options.dataRoot,
|
||||
options.workspaceId,
|
||||
options.workspaceRevision,
|
||||
options.effectiveRelationshipsPath,
|
||||
),
|
||||
runtimePaths: runtimePaths(options.dataRoot, options.workspaceId),
|
||||
installationOverlay: overlay,
|
||||
bindings,
|
||||
bindingDigest: stableBindingDigest(bindings),
|
||||
semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
|
||||
releaseSecrets: () => secretLease?.release(),
|
||||
releaseSecrets: () => {
|
||||
catalogLease?.release();
|
||||
secretLease?.release();
|
||||
},
|
||||
renderedConfig: renderRuntimeConfig(
|
||||
options.workspace,
|
||||
runtimeWorkspace,
|
||||
bindings,
|
||||
runtimePaths(
|
||||
options.dataRoot,
|
||||
options.workspaceId,
|
||||
options.workspaceRevision,
|
||||
options.effectiveRelationshipsPath,
|
||||
),
|
||||
runtimePaths(options.dataRoot, options.workspaceId),
|
||||
context,
|
||||
overlay,
|
||||
options.semanticRuntime,
|
||||
),
|
||||
};
|
||||
} catch (error) {
|
||||
catalogLease?.release();
|
||||
secretLease?.release();
|
||||
throw error;
|
||||
}
|
||||
@@ -372,8 +374,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
|
||||
dataRoot: string;
|
||||
secretRoots: readonly string[];
|
||||
semanticRuntime: SemanticRuntimeConfig;
|
||||
effectiveRelationshipsPath?: string;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
catalogDatabase?: WorkspaceDatabase;
|
||||
}): RenderedWorkspaceRuntime {
|
||||
const snapshot = readSnapshotWorkspace(options.snapshotPath);
|
||||
return renderWorkspaceRuntimeFromWorkspace({
|
||||
@@ -386,8 +388,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
|
||||
dataRoot: options.dataRoot,
|
||||
secretRoots: options.secretRoots,
|
||||
semanticRuntime: options.semanticRuntime,
|
||||
effectiveRelationshipsPath: options.effectiveRelationshipsPath,
|
||||
workspaceSecretStore: options.workspaceSecretStore,
|
||||
catalogDatabase: options.catalogDatabase,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -401,6 +403,7 @@ export async function renderActiveWorkspaceRuntime(options: {
|
||||
secretRoots: readonly string[];
|
||||
semanticRuntime: SemanticRuntimeConfig;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
catalogDatabase?: WorkspaceDatabase;
|
||||
}): Promise<ActiveRenderedWorkspaceRuntime> {
|
||||
// The persisted active state may reference host-side snapshot paths (written by another
|
||||
// process or installation). Read the active state directly and resolve the immutable snapshot
|
||||
@@ -431,6 +434,7 @@ export async function renderActiveWorkspaceRuntime(options: {
|
||||
secretRoots: options.secretRoots,
|
||||
semanticRuntime: options.semanticRuntime,
|
||||
workspaceSecretStore: options.workspaceSecretStore,
|
||||
catalogDatabase: options.catalogDatabase,
|
||||
});
|
||||
return {
|
||||
...rendered,
|
||||
@@ -480,6 +484,7 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
secretRoots: readonly string[];
|
||||
semanticRuntime: SemanticRuntimeConfig;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
catalogDatabase?: WorkspaceDatabase;
|
||||
}): Promise<DeterministicRuntimeConfigLease> {
|
||||
const rendered = await renderActiveWorkspaceRuntime(options);
|
||||
const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
|
||||
@@ -487,8 +492,15 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
|
||||
const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
|
||||
const configFingerprintValue = configFingerprint(renderedConfigObject);
|
||||
const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject);
|
||||
const identitySuffix = inputFingerprintValue.slice(7, 23);
|
||||
const inputFingerprintValue = preprocessingInputFingerprint(
|
||||
rendered.workspaceId,
|
||||
rendered.workspaceRevision,
|
||||
renderedConfigObject,
|
||||
);
|
||||
// The Catalog input identity intentionally excludes representation-only changes.
|
||||
// A lease also identifies its bytes, so a new renderer never collides with an
|
||||
// immutable config produced by an earlier release for the same Catalog inputs.
|
||||
const identitySuffix = sha256(inputFingerprintValue + "\n" + publishedConfig).slice(7, 23);
|
||||
|
||||
const preprocessingRoot = ensureTrustedDirectory(join(
|
||||
options.dataRoot,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { basename, join } from "node:path";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { stringify } from "yaml";
|
||||
import { buildInstallationContract } from "./contracts.js";
|
||||
import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js";
|
||||
import type { ResolvedBinding, ResolvedEvidenceBinding, RuntimeBindings } from "./bindings.js";
|
||||
import { workspaceVectorCollections } from "./vector-collections.js";
|
||||
export type { RuntimeBindings } from "./bindings.js";
|
||||
|
||||
export interface RuntimePaths {
|
||||
@@ -10,10 +11,8 @@ export interface RuntimePaths {
|
||||
artifacts: string;
|
||||
indexes: string;
|
||||
memory: string;
|
||||
/** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */
|
||||
annotations_root?: string;
|
||||
/** Immutable Catalog projection used as the exclusive runtime relationship source. */
|
||||
effective_relationships?: string;
|
||||
/** Current backend-produced projection of the PostgreSQL Metadata Catalog. */
|
||||
catalog_metadata_snapshot?: string;
|
||||
}
|
||||
|
||||
export interface RuntimeIdentity {
|
||||
@@ -180,6 +179,7 @@ function renderEvidence(
|
||||
return {
|
||||
evidence: {
|
||||
...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}),
|
||||
local_archive_root: join(dirname(dirname(context.revisionContentRoot)), "repo", context.workspaceId),
|
||||
sources: [renderedSource],
|
||||
},
|
||||
vector: {
|
||||
@@ -224,6 +224,7 @@ export function renderRuntimeConfig(
|
||||
): string {
|
||||
requireSupportedDescriptor(workspace);
|
||||
const descriptor = validateWorkspaceDescriptor(workspace);
|
||||
if (!descriptor.dwh) throw new Error("runtime configuration requires a Catalog database binding");
|
||||
const contract = buildInstallationContract(descriptor);
|
||||
const name = (role: "DWH" | "EVIDENCE", suffix: string) => {
|
||||
const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix);
|
||||
@@ -243,6 +244,7 @@ export function renderRuntimeConfig(
|
||||
}
|
||||
|
||||
const dwhIdentity = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
|
||||
const vectorCollections = workspaceVectorCollections(descriptor.workspace.id);
|
||||
const database = bindings.dwh.transport === "postgres_direct"
|
||||
? { ...directConnection(bindings.dwh, {
|
||||
host: name("DWH", "HOST"),
|
||||
@@ -268,7 +270,7 @@ export function renderRuntimeConfig(
|
||||
semantic_index: {
|
||||
vector_store: {
|
||||
engine: "qdrant",
|
||||
collection: descriptor.workspace.id,
|
||||
collections: vectorCollections,
|
||||
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
||||
distance: "cosine",
|
||||
},
|
||||
@@ -284,7 +286,7 @@ export function renderRuntimeConfig(
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
base_url: semanticRuntime.internalQdrantUrl,
|
||||
collection: descriptor.workspace.id,
|
||||
collections: vectorCollections,
|
||||
},
|
||||
embeddings: {
|
||||
provider: "ollama_internal",
|
||||
|
||||
@@ -53,7 +53,8 @@ interface WorkspaceDwh {
|
||||
|
||||
interface WorkspaceBase {
|
||||
workspace: WorkspaceMetadata;
|
||||
dwh: WorkspaceDwh;
|
||||
/** Legacy connection block; current descriptors bind their database through PostgreSQL. */
|
||||
dwh?: WorkspaceDwh;
|
||||
diagnostics?: Pick<CanonicalDiagnostics, "dwh_rest">;
|
||||
}
|
||||
|
||||
@@ -328,7 +329,9 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert
|
||||
}
|
||||
|
||||
function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
|
||||
unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
|
||||
if (workspace.dwh) {
|
||||
unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
|
||||
}
|
||||
|
||||
if (workspace.evidence?.source.type === "filesystem") {
|
||||
const expected = `${workspace.workspace.id}/evidence`;
|
||||
@@ -341,7 +344,8 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
|
||||
}
|
||||
}
|
||||
|
||||
if (workspace.diagnostics?.dwh_rest && !workspace.dwh.supported_transports.includes("rest_api")) {
|
||||
if (workspace.diagnostics?.dwh_rest
|
||||
&& !workspace.dwh?.supported_transports.includes("rest_api")) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["diagnostics", "dwh_rest"],
|
||||
@@ -351,7 +355,7 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
|
||||
}
|
||||
|
||||
const WorkspaceV4Schema = z.object({
|
||||
dwh: dwhSchema,
|
||||
dwh: dwhSchema.optional(),
|
||||
evidence: workspaceEvidenceSchema.optional(),
|
||||
diagnostics: z.object({
|
||||
dwh_rest: dwhRestDiagnostic.optional(),
|
||||
@@ -363,7 +367,7 @@ const WorkspaceV4Schema = z.object({
|
||||
}).strict().superRefine(workspaceInvariants);
|
||||
const WorkspaceDescriptorSchema = WorkspaceV4Schema;
|
||||
|
||||
export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
|
||||
function parseWorkspaceDocument(source: string): unknown {
|
||||
const documents = parseAllDocuments(source, { uniqueKeys: true });
|
||||
if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
|
||||
const document = documents[0];
|
||||
@@ -371,10 +375,30 @@ export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
|
||||
throw new Error(`Invalid workspace YAML: ${[...document.errors, ...document.warnings]
|
||||
.map((error) => error.message).join("; ")}`);
|
||||
}
|
||||
return validateWorkspaceDescriptor(document.toJSON());
|
||||
return document.toJSON();
|
||||
}
|
||||
|
||||
/** Deterministically removes the two installation-owned v3 blocks without altering workspace data. */
|
||||
export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
|
||||
const value = parseWorkspaceDocument(source);
|
||||
assertAuthoredWorkspaceIsDatabaseFree(value);
|
||||
return validateWorkspaceDescriptor(value);
|
||||
}
|
||||
|
||||
/** Parses an ephemeral, generated core runtime descriptor that may contain a Catalog binding. */
|
||||
export function parseRuntimeWorkspaceYaml(source: string): WorkspaceDescriptor {
|
||||
return validateWorkspaceDescriptor(parseWorkspaceDocument(source));
|
||||
}
|
||||
|
||||
function assertAuthoredWorkspaceIsDatabaseFree(workspace: unknown): void {
|
||||
if (workspace !== null && typeof workspace === "object"
|
||||
&& ("dwh" in workspace || "diagnostics" in workspace)) {
|
||||
throw new Error(
|
||||
"Workspace YAML must not contain database configuration; use the PostgreSQL Metadata Catalog",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a database-free v4 descriptor; legacy database/configuration fields are not carried over. */
|
||||
export function migrateWorkspaceV3Yaml(source: string): string {
|
||||
const documents = parseAllDocuments(source, { uniqueKeys: true });
|
||||
if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
|
||||
@@ -388,6 +412,8 @@ export function migrateWorkspaceV3Yaml(source: string): string {
|
||||
throw new Error("Workspace migration requires schema version 3");
|
||||
}
|
||||
metadata.schema_version = 4;
|
||||
delete value.dwh;
|
||||
delete value.diagnostics;
|
||||
delete value.semantic_index;
|
||||
delete value.llm_policy;
|
||||
return serializeWorkspaceYaml(validateWorkspaceDescriptor(value));
|
||||
@@ -423,6 +449,7 @@ export function resolveDiagnosticUrl(baseUrl: string, path: string): URL {
|
||||
|
||||
export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string {
|
||||
const canonical = validateOperationalWorkspace(workspace);
|
||||
assertAuthoredWorkspaceIsDatabaseFree(canonical);
|
||||
return stringify(canonical, { lineWidth: 0, sortMapEntries: true });
|
||||
}
|
||||
|
||||
|
||||
@@ -101,7 +101,8 @@ function selectedTransport(
|
||||
descriptor: WorkspaceDescriptor,
|
||||
variables: readonly InstallationVariable[],
|
||||
env: NodeJS.ProcessEnv,
|
||||
): DwhTransport {
|
||||
): DwhTransport | undefined {
|
||||
if (!descriptor.dwh) return undefined;
|
||||
const transportVariable = variables.find(({ role, suffix }) => role === "DWH" && suffix === "TRANSPORT");
|
||||
const value = transportVariable === undefined ? undefined : env[transportVariable.name];
|
||||
return isTransport(value) && descriptor.dwh.supported_transports.includes(value)
|
||||
@@ -136,11 +137,14 @@ export function discoverWorkspaceSecretRequirements(
|
||||
const variables = buildInstallationContract(descriptor).variables;
|
||||
const transport = selectedTransport(descriptor, variables, env);
|
||||
const restHasNoAuthentication = transport === "rest_api" && descriptor.diagnostics?.dwh_rest?.auth === "none";
|
||||
const requiredDwh = new Set(restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport]);
|
||||
const requiredDwh = new Set(
|
||||
transport === undefined || restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport],
|
||||
);
|
||||
|
||||
const requirements: WorkspaceSecretRequirement[] = [];
|
||||
for (const variable of variables) {
|
||||
if (variable.role === "DWH") {
|
||||
if (transport === undefined) continue;
|
||||
if (variable.transports !== undefined && !variable.transports.includes(transport)) continue;
|
||||
const requirement = requirementFor(variable, requiredDwh.has(variable.suffix));
|
||||
if (requirement !== undefined && requirement.required) requirements.push(requirement);
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
export interface WorkspaceVectorCollections {
|
||||
reference: string;
|
||||
memory: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Physical Qdrant namespaces owned by one workspace.
|
||||
*
|
||||
* Reference data is replaceable preprocessing output. Memory is durable runtime
|
||||
* state and deliberately has a separate lifecycle.
|
||||
*/
|
||||
export function workspaceVectorCollections(workspaceId: string): WorkspaceVectorCollections {
|
||||
if (!/^[a-z][a-z0-9-]{2,62}$/.test(workspaceId)) {
|
||||
throw new Error("workspace id is invalid");
|
||||
}
|
||||
return {
|
||||
reference: `${workspaceId}-reference`,
|
||||
memory: `${workspaceId}-memory`,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import Fastify from "fastify";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { sessionRoutes } from "../src/routes/sessions.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
test.each([
|
||||
[false, "m", 403], [false, "e", 403], [false, "reject", 204],
|
||||
[true, "m", 204], [true, "e", 204], [true, "forged", 400],
|
||||
] as const)("archive repair response enforces the responding principal (%s, %s)", async (admin, choice, status) => {
|
||||
const app = Fastify();
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "reviewer", isAdmin: admin,
|
||||
roles: admin ? ["admin"] : ["user"],
|
||||
permissions: admin ? ["session.use", "memory.manage", "evidence.manage"] : ["session.use"] };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const respond = vi.fn(() => true);
|
||||
sessionRoutes(app, {
|
||||
tht: {}, mgr: { get: () => ({ ownerKey: "test\0reviewer", bridge: {
|
||||
respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair", repair: { options: [
|
||||
{ id: "m", archive: "memory" }, { id: "e", archive: "evidence" },
|
||||
] } }),
|
||||
} }) },
|
||||
} as unknown as Parameters<typeof sessionRoutes>[1]);
|
||||
try {
|
||||
const result = await app.inject({ method: "POST", url: "/sessions/s/response",
|
||||
payload: { ui_response: { id: "gate", choices: [choice] } } });
|
||||
expect(result.statusCode).toBe(status);
|
||||
expect(respond).toHaveBeenCalledTimes(status === 204 ? 1 : 0);
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("an administrator cannot attribute a repair to another runtime's principal", async () => {
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async request => { request.principal = {
|
||||
issuer: "test", subject: "second-admin", isAdmin: true, roles: ["admin"],
|
||||
permissions: ["session.use", "memory.manage"],
|
||||
}; });
|
||||
const respond = vi.fn();
|
||||
sessionRoutes(app, { tht: {}, mgr: { get: () => ({ ownerKey: "test\0first-admin", bridge: {
|
||||
respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair",
|
||||
repair: { options: [{ id: "m", archive: "memory" }] } }),
|
||||
} }) } } as unknown as Parameters<typeof sessionRoutes>[1]);
|
||||
try {
|
||||
const result = await app.inject({ method: "POST", url: "/sessions/s/response",
|
||||
payload: { ui_response: { id: "gate", choices: ["m"] } } });
|
||||
expect(result.statusCode).toBe(409);
|
||||
expect(respond).not.toHaveBeenCalled();
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
@@ -188,6 +188,8 @@ test("roles collapse duplicates and admin contains all administrative permission
|
||||
"workspace.manage",
|
||||
"workspace.secrets.manage",
|
||||
"database.manage",
|
||||
"memory.manage",
|
||||
"evidence.manage",
|
||||
"pi.manage",
|
||||
"auth.diagnostics.read",
|
||||
]);
|
||||
|
||||
@@ -130,7 +130,7 @@ test("local login sets a non-persistent opaque session cookie and exposes only a
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/),
|
||||
|
||||
@@ -7,6 +7,8 @@ import { join } from "node:path";
|
||||
import { expandLocalHome, localPrincipal, upstreamPrincipal } from "../src/auth/principal.js";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { rolesToPermissions } from "../src/auth/config.js";
|
||||
import type { Permission, Role } from "../src/auth/types.js";
|
||||
|
||||
test("server smoke rejects retired trusted claims under OIDC authentication", () => {
|
||||
const smoke = readFileSync("../scripts/unified-deployment-smoke.sh", "utf8");
|
||||
@@ -32,7 +34,7 @@ test("local mode resolves a stable local principal", async () => {
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -74,7 +76,7 @@ test("upstream mode accepts only normalized proxy principal headers", async () =
|
||||
issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -206,15 +208,33 @@ test("the session boundary rejects tht maintenance headers outside exact loopbac
|
||||
})).statusCode).toBe(503);
|
||||
});
|
||||
|
||||
test("the session boundary touches a valid cookie session through the bounded Task 7 store operation", async () => {
|
||||
test.each<{
|
||||
name: string;
|
||||
roles: Role[];
|
||||
storedPermissions: Permission[];
|
||||
}>([
|
||||
{ name: "current user", roles: ["user"], storedPermissions: ["session.use"] },
|
||||
{
|
||||
name: "administrator signed in before archive permissions existed",
|
||||
roles: ["admin"],
|
||||
storedPermissions: rolesToPermissions(["admin"]).filter(
|
||||
(permission) => permission !== "memory.manage" && permission !== "evidence.manage",
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "user with obsolete administrative permissions",
|
||||
roles: ["user"],
|
||||
storedPermissions: ["session.use", "memory.manage", "evidence.manage"],
|
||||
},
|
||||
])("the session boundary derives current permissions and touches a valid cookie: $name", async ({ roles, storedPermissions }) => {
|
||||
const sessions = {
|
||||
resolve: vi.fn(async () => ({
|
||||
version: 1,
|
||||
issuer: "local",
|
||||
subject: "user-1",
|
||||
method: "local",
|
||||
roles: ["user"],
|
||||
permissions: ["session.use"],
|
||||
roles,
|
||||
permissions: storedPermissions,
|
||||
userAuthRevision: 1,
|
||||
authConfigRevision: "b".repeat(64),
|
||||
remembered: false,
|
||||
@@ -249,7 +269,13 @@ test("the session boundary touches a valid cookie session through the bounded Ta
|
||||
app.get("/private", async (request) => getPrincipal(request));
|
||||
|
||||
const token = "z".repeat(43);
|
||||
expect((await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } })).statusCode).toBe(200);
|
||||
const response = await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({
|
||||
roles,
|
||||
permissions: rolesToPermissions(roles),
|
||||
isAdmin: roles.includes("admin"),
|
||||
});
|
||||
expect(sessions.touch).toHaveBeenCalledWith(token);
|
||||
});
|
||||
|
||||
|
||||
@@ -97,31 +97,23 @@ const fleetSnapshot: ObservedSchemaSnapshot = {
|
||||
}],
|
||||
};
|
||||
|
||||
test("lists every YAML workspace and creates its one database configuration", async () => {
|
||||
test("lists every workspace and creates its Catalog database configuration", async () => {
|
||||
const { app, secretStore } = setup();
|
||||
const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(initial.statusCode).toBe(200);
|
||||
expect(initial.json()).toMatchObject([{
|
||||
workspaceId: "psd-clinical",
|
||||
configured: false,
|
||||
databaseName: "warehouse",
|
||||
databaseName: "",
|
||||
workspaceRevision: { commit: revision.commit, blob: revision.blob },
|
||||
workspaceEvidence: { sourceType: null, state: "not_declared" },
|
||||
runtimeBinding: {
|
||||
transport: "postgres_direct",
|
||||
configurationState: "configuration_required",
|
||||
sessionTransportSupported: true,
|
||||
},
|
||||
runtimeBinding: null,
|
||||
}]);
|
||||
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
|
||||
secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" });
|
||||
const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(runtimeReady.json()).toMatchObject([{
|
||||
runtimeBinding: { configurationState: "ready", sessionTransportSupported: true },
|
||||
runtimeBinding: null,
|
||||
}]);
|
||||
|
||||
const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
|
||||
@@ -166,7 +158,7 @@ test("projects remote Evidence credential state without conflating catalog secre
|
||||
}]);
|
||||
});
|
||||
|
||||
test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
|
||||
test("lists orphaned records and keeps the REST diagnostic path in the Catalog", async () => {
|
||||
const { app, repository } = setup();
|
||||
await repository.create({
|
||||
workspaceId: "removed-workspace",
|
||||
@@ -186,7 +178,7 @@ test("lists orphaned records and takes the REST diagnostic path from workspace Y
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(201);
|
||||
expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
|
||||
expect(created.json()).toMatchObject({ binding: { restPath: "/client-controlled" } });
|
||||
|
||||
const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
|
||||
expect(rows).toEqual(expect.arrayContaining([
|
||||
|
||||
@@ -577,7 +577,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
|
||||
expect.objectContaining({
|
||||
sequence: 3,
|
||||
level: "info",
|
||||
message: `Generated description for Catalog Column ${column.id}.`,
|
||||
message: 'Generated description for Column "patients.birth_date".',
|
||||
}),
|
||||
expect.objectContaining({ sequence: 4, level: "info", message: "Description generation completed." }),
|
||||
]);
|
||||
@@ -934,7 +934,10 @@ test("generates selected Catalog Columns in caller order through sequential batc
|
||||
const events = await repository.listDescriptionGenerationEvents(run.id);
|
||||
expect(events.map((event) => event.sequence)).toEqual(Array.from({ length: 14 }, (_, index) => index + 1));
|
||||
expect(events.slice(2, -1).map((event) => event.message)).toEqual(
|
||||
orderedIds.map((targetId) => `Generated description for Catalog Column ${targetId}.`),
|
||||
orderedIds.map((targetId) => {
|
||||
const target = columns.find((column) => column.id === targetId)!;
|
||||
return `Generated description for Column "patients.${target.name}".`;
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
pending[0]!.resolve(responseFor(orderedIds.slice(0, 10), 0));
|
||||
@@ -1803,7 +1806,9 @@ test("retains completed batch writes when a later batch response is malformed",
|
||||
});
|
||||
const events = await repository.listDescriptionGenerationEvents(run.id);
|
||||
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
|
||||
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
|
||||
originalColumns.slice(0, 10).map(
|
||||
(target) => `Generated description for Column "patients.${target.name}".`,
|
||||
),
|
||||
);
|
||||
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
|
||||
expect.objectContaining({
|
||||
@@ -1812,7 +1817,7 @@ test("retains completed batch writes when a later batch response is malformed",
|
||||
);
|
||||
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
|
||||
message: `The model response did not match the required schema. Affected target: Column "patients.${originalColumns[10]!.name}".`,
|
||||
}));
|
||||
} finally {
|
||||
await app.close();
|
||||
@@ -2301,7 +2306,7 @@ test("generates selected Catalog Tables with structural column context and local
|
||||
expect((await repository.listDescriptionGenerationEvents(run.id)).map((event) => event.message)).toEqual([
|
||||
"Description generation queued.",
|
||||
"Description generation started.",
|
||||
`Stored non-generatable result for Catalog Table ${table.id}.`,
|
||||
'Stored non-generatable result for Table "patients".',
|
||||
"Description generation completed.",
|
||||
]);
|
||||
} finally {
|
||||
@@ -2445,7 +2450,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
|
||||
expect(`${JSON.stringify(run)}${events.body}`).not.toContain(sensitiveDiagnostic);
|
||||
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model provider request failed. Affected Catalog Column target: ${column.id}.`,
|
||||
message: 'The model provider request failed. Affected target: Column "patients.birth_date".',
|
||||
}));
|
||||
} finally {
|
||||
await app.close();
|
||||
@@ -2536,7 +2541,7 @@ test.each([
|
||||
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
|
||||
expect.objectContaining({
|
||||
level: "error",
|
||||
message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
|
||||
message: `${failureMessage} Affected targets: Column "patients.first_column", Column "patients.second_column".`,
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
|
||||
@@ -13,10 +13,12 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
|
||||
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
||||
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
||||
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
||||
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
|
||||
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
||||
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
||||
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
|
||||
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
|
||||
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
|
||||
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
@@ -52,10 +54,12 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
||||
await upSensitiveDataFlag(db);
|
||||
await upDescriptionGeneration(db);
|
||||
await upSensitiveSuggestionRuns(db);
|
||||
await upLogicalRelationships(db);
|
||||
await upAiTokenUsage(db);
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
await upSensitivityReason(db);
|
||||
await upCatalogPreprocessingState(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -282,7 +286,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
||||
expect(events.statusCode).toBe(200);
|
||||
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
|
||||
message: 'The model provider request failed. Affected target: Column "patients.status".',
|
||||
}));
|
||||
expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usa
|
||||
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
||||
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
|
||||
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
|
||||
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
@@ -58,6 +59,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
await upSensitivityReason(db);
|
||||
await upCatalogPreprocessingState(db);
|
||||
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
|
||||
.select(["engine", "modelId", "policyVersion", "unknown"])
|
||||
.where("id", "=", historicalSuggestionRunId)
|
||||
@@ -122,8 +124,17 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
],
|
||||
relationships: [],
|
||||
};
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot))
|
||||
const memoryCleanupRun = await repository.createSyncRun(created.id, "columns", [], 1);
|
||||
await repository.updateSyncRun(memoryCleanupRun.id, { state: "applying", phase: "applying" });
|
||||
await expect(repository.applySchemaSync(created.id, 999, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
||||
.resolves.toBeUndefined();
|
||||
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("applying");
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
||||
.toMatchObject({ created: 4, deleted: 0 });
|
||||
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("memory_cleanup");
|
||||
await repository.interruptActiveSyncRuns();
|
||||
expect(await repository.getSyncRun(memoryCleanupRun.id))
|
||||
.toMatchObject({ state: "interrupted", phase: "memory_cleanup" });
|
||||
expect((await repository.listColumns(created.id, patients.id)).map((column) => ({
|
||||
name: column.name,
|
||||
sensitive: column.sensitive,
|
||||
@@ -231,7 +242,79 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
});
|
||||
expect(await repository.listSyncRuns(created.id)).toEqual([
|
||||
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
||||
expect.objectContaining({ id: memoryCleanupRun.id, phase: "memory_cleanup" }),
|
||||
]);
|
||||
|
||||
const lockedDatabase = await repository.create({
|
||||
workspaceId: "preprocessing-lock",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "public",
|
||||
binding: {
|
||||
transport: "postgres_direct", host: "lock.internal", port: 5432, username: "reader",
|
||||
},
|
||||
});
|
||||
await repository.applySchemaSync(
|
||||
lockedDatabase.id,
|
||||
lockedDatabase.version,
|
||||
"all",
|
||||
[],
|
||||
{
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [], columns: [], relationships: [],
|
||||
},
|
||||
);
|
||||
const beforePreprocessing = (await repository.get(lockedDatabase.id))!;
|
||||
const preprocessing = await repository.beginPreprocessing(
|
||||
"preprocessing-lock",
|
||||
"sha256:" + "1".repeat(64),
|
||||
);
|
||||
expect(preprocessing).toMatchObject({ kind: "started" });
|
||||
await expect(db.insertInto("catalogTables").values({
|
||||
id: randomUUID(),
|
||||
databaseId: lockedDatabase.id,
|
||||
name: "blocked_write",
|
||||
sourceComment: null,
|
||||
description: null,
|
||||
generatedDescription: null,
|
||||
}).execute()).rejects.toThrow("catalog preprocessing is running");
|
||||
await expect(repository.createDescriptionGenerationRun(
|
||||
lockedDatabase.id,
|
||||
"all",
|
||||
"openai/gpt-5-mini",
|
||||
"en",
|
||||
0,
|
||||
)).rejects.toThrow("catalog preprocessing is running");
|
||||
await repository.recordTest(lockedDatabase.id, lockedDatabase.version, {
|
||||
connectionStatus: "reachable",
|
||||
testedVersion: lockedDatabase.version,
|
||||
lastTestedAt: "2026-01-01T00:00:00Z",
|
||||
});
|
||||
expect((await repository.get(lockedDatabase.id))?.metadataContentRevision)
|
||||
.toBe(beforePreprocessing.metadataContentRevision);
|
||||
await expect(repository.finishPreprocessing(
|
||||
"preprocessing-lock",
|
||||
beforePreprocessing.metadataContentRevision,
|
||||
"sha256:" + "1".repeat(64),
|
||||
{ status: "succeeded" },
|
||||
)).resolves.toMatchObject({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: beforePreprocessing.metadataContentRevision,
|
||||
});
|
||||
await db.insertInto("catalogTables").values({
|
||||
id: randomUUID(),
|
||||
databaseId: lockedDatabase.id,
|
||||
name: "allowed_after_preprocessing",
|
||||
sourceComment: null,
|
||||
description: null,
|
||||
generatedDescription: null,
|
||||
}).execute();
|
||||
await expect(repository.get(lockedDatabase.id)).resolves.toMatchObject({
|
||||
preprocessingStatus: "failed",
|
||||
metadataContentRevision: beforePreprocessing.metadataContentRevision + 1,
|
||||
});
|
||||
expect(await repository.delete(lockedDatabase.id, lockedDatabase.version)).toBe(true);
|
||||
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
|
||||
expect(await repository.delete(created.id, 2)).toBe(true);
|
||||
expect(await repository.list()).toEqual([]);
|
||||
@@ -355,7 +438,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected and database-wide descriptions", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
@@ -449,6 +532,22 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel
|
||||
description: "Still curated visits",
|
||||
generatedDescription: "Generated visits",
|
||||
});
|
||||
|
||||
expect(await repository.consolidateGeneratedDescriptions(
|
||||
database.id, "database", [],
|
||||
)).toEqual({ copied: 3, skipped: 1 });
|
||||
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
||||
description: "Generated visits",
|
||||
generatedDescription: "Generated visits",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
||||
description: "Generated id",
|
||||
generatedDescription: "Generated id",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
||||
description: "Keep patient reference",
|
||||
generatedDescription: null,
|
||||
});
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
@@ -473,6 +572,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
await upSensitivityReason(db);
|
||||
await upCatalogPreprocessingState(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const firstDatabase = await repository.create({
|
||||
workspaceId: "generation-one",
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
|
||||
import { resolveCatalogRuntimeBinding } from "../src/catalog/runtime-binding.js";
|
||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("projects a Catalog database into a runtime without database data in workspace YAML", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "thoth-catalog-runtime-"));
|
||||
roots.push(root);
|
||||
const secretStore = new WorkspaceSecretStore({
|
||||
root: join(root, "vault"),
|
||||
runtimeRoot: join(root, "runtime"),
|
||||
installationId: "test",
|
||||
});
|
||||
secretStore.putMany("sales", {
|
||||
"catalog.dwh.password": "catalog-password",
|
||||
"evidence.signed_urls": '["https://signed.example.test/evidence"]',
|
||||
});
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: {
|
||||
schema_version: 4,
|
||||
id: "sales",
|
||||
name: "Sales",
|
||||
language: "en",
|
||||
},
|
||||
evidence: {
|
||||
schema_version: 1,
|
||||
source: {
|
||||
type: "http",
|
||||
uris: ["https://evidence.example.test/guide.md"],
|
||||
authentication: "signed_urls_file",
|
||||
connect_timeout_ms: 1_000,
|
||||
read_timeout_ms: 5_000,
|
||||
max_bytes: 10_000,
|
||||
max_redirects: 2,
|
||||
allow_private_hosts: false,
|
||||
max_cache_bytes: 20_000,
|
||||
},
|
||||
policy: { max_chunk_chars: 4_000, retain_published_generations: 1 },
|
||||
},
|
||||
};
|
||||
const database: WorkspaceDatabase = {
|
||||
id: "database-1",
|
||||
workspaceId: "sales",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "analytics",
|
||||
version: 3,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
binding: {
|
||||
transport: "postgres_direct",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
connectionStatus: "reachable",
|
||||
metadataContentRevision: 7,
|
||||
preprocessingStatus: "failed",
|
||||
};
|
||||
|
||||
const lease = resolveCatalogRuntimeBinding({
|
||||
workspace,
|
||||
database,
|
||||
environment: {},
|
||||
secretRoots: [],
|
||||
secretStore,
|
||||
});
|
||||
const passwordPath = lease.bindings.dwh.values.THT_WS_SALES_DWH_PASSWORD_FILE;
|
||||
const evidencePath = lease.bindings.evidence.values.THT_WS_SALES_EVIDENCE_SIGNED_URLS_FILE;
|
||||
try {
|
||||
expect(workspace.dwh).toBeUndefined();
|
||||
expect(lease.workspace.dwh).toMatchObject({
|
||||
database: "warehouse",
|
||||
schema: "analytics",
|
||||
supported_transports: ["postgres_direct"],
|
||||
});
|
||||
expect(lease.bindings.dwh).toMatchObject({
|
||||
transport: "postgres_direct",
|
||||
missing: [],
|
||||
values: {
|
||||
THT_WS_SALES_DWH_HOST: "db.internal",
|
||||
THT_WS_SALES_DWH_PORT: "5432",
|
||||
THT_WS_SALES_DWH_USER: "reader",
|
||||
},
|
||||
});
|
||||
expect(readFileSync(passwordPath, "utf8")).toBe("catalog-password");
|
||||
expect(readFileSync(evidencePath, "utf8")).toContain("signed.example.test");
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
expect(existsSync(passwordPath)).toBe(false);
|
||||
expect(existsSync(evidencePath)).toBe(false);
|
||||
});
|
||||
@@ -117,6 +117,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
});
|
||||
const introspector: CatalogSchemaIntrospector = { scan };
|
||||
const operations = new CatalogOperationCoordinator();
|
||||
const cleanup = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 0 }), stderr: "" }));
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
@@ -124,7 +125,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), {
|
||||
thtRunner: {} as never,
|
||||
thtRunner: { withPrincipal: () => ({ runWithRuntimeSnapshot: cleanup }) } as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
||||
catalogRepository: repository,
|
||||
@@ -133,7 +134,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return {
|
||||
app, repository, database: (await repository.get(created.id))!, scan, operations,
|
||||
app, repository, database: (await repository.get(created.id))!, scan, operations, cleanup,
|
||||
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
||||
};
|
||||
}
|
||||
@@ -394,12 +395,19 @@ test("consolidates non-empty generated column descriptions and preserves curated
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("consolidates generated descriptions for every column in a database", async () => {
|
||||
test("consolidates generated descriptions for every table and column in a database", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const tables = await repository.listTables(database.id);
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
await repository.updateTableMetadata(
|
||||
database.id,
|
||||
patients.id,
|
||||
patients.version,
|
||||
"Curated patients",
|
||||
"Generated patients",
|
||||
);
|
||||
const patientId = (await repository.listColumns(database.id, patients.id))[0]!;
|
||||
const visitColumns = await repository.listColumns(database.id, visits.id);
|
||||
const visitId = visitColumns.find((column) => column.name === "id")!;
|
||||
@@ -432,11 +440,16 @@ test("consolidates generated descriptions for every column in a database", async
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "database_columns" },
|
||||
payload: { target: "database" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({ copied: 2, skipped: 1 });
|
||||
expect(response.json()).toEqual({ copied: 3, skipped: 2 });
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
||||
description: "Generated patients",
|
||||
generatedDescription: "Generated patients",
|
||||
version: patients.version + 2,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({
|
||||
description: "Generated patient identifier",
|
||||
generatedDescription: "Generated patient identifier",
|
||||
@@ -529,6 +542,11 @@ test("strictly validates description consolidation database and target ids", asy
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "tables", targetIds: [table.id], unexpected: true },
|
||||
}),
|
||||
app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "database", targetIds: [table.id] },
|
||||
}),
|
||||
app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
@@ -536,7 +554,7 @@ test("strictly validates description consolidation database and target ids", asy
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400]);
|
||||
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400, 400]);
|
||||
for (const response of responses) {
|
||||
expect(response.json()).toEqual({
|
||||
code: "description_consolidation_invalid",
|
||||
@@ -597,6 +615,45 @@ test("waits for confirmation and rescans before applying destructive changes", a
|
||||
expect(scan).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
test("retries committed Memory cleanup with the original removals without rescanning", async () => {
|
||||
const { app, repository, database, scan, setObserved, cleanup } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const observed = snapshot();
|
||||
observed.tables = observed.tables.filter(t => t.name !== "visits");
|
||||
observed.columns = observed.columns.filter(c => c.tableName !== "visits");
|
||||
observed.relationships = [];
|
||||
setObserved(observed);
|
||||
cleanup.mockResolvedValueOnce({ code: 0, stdout: JSON.stringify({ indexed: false, deleted: 2 }), stderr: "" });
|
||||
const started = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
const waiting = await waitFor(repository, started.json().id, "awaiting_confirmation");
|
||||
expect(cleanup).not.toHaveBeenCalled();
|
||||
await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`,
|
||||
payload: { confirmationToken: waiting.confirmationToken } });
|
||||
const failed = await waitFor(repository, waiting.id, "failed");
|
||||
expect(failed).toMatchObject({ phase: "memory_cleanup", errorCode: "memory_cleanup_pending",
|
||||
plannedDiff: { deletedTables: ["visits"] } });
|
||||
expect((await repository.listTables(database.id)).map(t => t.name)).toEqual(["patients"]);
|
||||
const callsBeforeRetry = scan.mock.calls.length;
|
||||
const blocked = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
expect(blocked.statusCode).toBe(409);
|
||||
// Simulate startup recovery and an unreachable DWH after schema application.
|
||||
await repository.interruptActiveSyncRuns();
|
||||
scan.mockRejectedValue(new Error("DWH unavailable"));
|
||||
cleanup.mockResolvedValue({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 2 }), stderr: "" });
|
||||
const retried = await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/retry` });
|
||||
expect(retried.statusCode).toBe(202);
|
||||
const completed = await waitFor(repository, waiting.id, "succeeded");
|
||||
expect(completed.counts).toMatchObject({ memoryDeleted: 2 });
|
||||
expect(scan.mock.calls.length).toBe(callsBeforeRetry);
|
||||
expect(cleanup).toHaveBeenCalledTimes(2);
|
||||
expect(cleanup.mock.calls[1]).toEqual(cleanup.mock.calls[0]);
|
||||
const request = JSON.parse((cleanup.mock.calls[0] as unknown as string[])[1]!).request;
|
||||
expect(request).toMatchObject({ sync_id: waiting.id, database: "warehouse",
|
||||
schema_name: "datawarehouse", removed_tables: ["visits"] });
|
||||
});
|
||||
|
||||
test("deletes every catalog table for multiple selected databases and cascades dependent metadata", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const second = await repository.create({
|
||||
|
||||
@@ -36,7 +36,7 @@ test("loop F1: crea sessione → SSE riceve il widget → risponde → il modell
|
||||
ollamaEnsure: async () => ({ ok: true }),
|
||||
searchPack: async () => {},
|
||||
sessionNew: async () => ({ id: "s1" }),
|
||||
sessionShow: async (_id: string) => ({ id: "s1", provider: undefined, model: undefined, thinking: undefined }),
|
||||
sessionShow: async (_id: string) => ({ id: "s1", interaction_language: "en", provider: undefined, model: undefined, thinking: undefined }),
|
||||
sessionList: async () => [],
|
||||
} as any,
|
||||
spawnFn: () => spawn("node", [FAKE, SCRIPT]) as any,
|
||||
@@ -48,7 +48,7 @@ test("loop F1: crea sessione → SSE riceve il widget → risponde → il modell
|
||||
const created = await fetch(`${base}/sessions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ workspace: "w", question: "q" }),
|
||||
body: JSON.stringify({ workspace: "w", question: "q", interactionLanguage: "en" }),
|
||||
});
|
||||
expect(created.status).toBe(200);
|
||||
|
||||
|
||||
@@ -41,7 +41,10 @@ function directRendered(): Record<string, unknown> {
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
base_url: "http://qdrant:6333",
|
||||
collection: "psd-clinical",
|
||||
collections: {
|
||||
reference: "psd-clinical-reference",
|
||||
memory: "psd-clinical-memory",
|
||||
},
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
collection_lifecycle: "require_existing",
|
||||
@@ -103,10 +106,10 @@ function restRendered(): Record<string, unknown> {
|
||||
}
|
||||
|
||||
const directCanonical =
|
||||
`{"schemaVersion":2,"dwh":{` +
|
||||
`{"schemaVersion":3,"dwh":{` +
|
||||
`"engine":"postgres","database":"postgres","schema":"datawarehouse",` +
|
||||
`"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` +
|
||||
`"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` +
|
||||
`"vector":{"collections":{"reference":"psd-clinical-reference","memory":"psd-clinical-memory"},"dimensions":1024,"distance":"cosine"},` +
|
||||
`"embedding":{"id":"ollama/qwen3-embedding:0.6b","model":"qwen3-embedding:0.6b","dimensions":1024},` +
|
||||
`"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` +
|
||||
`"indexes":"/data/sessions/psd-clinical/indexes"}}`;
|
||||
@@ -128,7 +131,7 @@ test("canonical effective config excludes secrets, evidence, session storage, an
|
||||
expect(json).not.toContain("sources");
|
||||
expect(json).not.toContain("collection_lifecycle");
|
||||
expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity
|
||||
expect(json).not.toContain("memory");
|
||||
expect(json).not.toContain('"memory":"/data');
|
||||
expect(json).not.toContain('"sessions"');
|
||||
});
|
||||
|
||||
@@ -190,7 +193,16 @@ test("DWH-affecting changes alter the effective config identity", () => {
|
||||
const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } };
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore);
|
||||
|
||||
const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record<string, any>).vector, collection: "other" } } };
|
||||
const changedCollection = {
|
||||
...base,
|
||||
resources: {
|
||||
...base.resources,
|
||||
vector: {
|
||||
...(base.resources as Record<string, any>).vector,
|
||||
collections: { reference: "other-reference", memory: "other-memory" },
|
||||
},
|
||||
},
|
||||
};
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore);
|
||||
|
||||
const changedEmbeddingIdentity = { ...base, resources: { ...base.resources, embeddings: { ...(base.resources as Record<string, any>).embeddings, model: "other-embedding" } } };
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import Fastify from "fastify";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { evidenceRoutes } from "../src/routes/evidence.js";
|
||||
import type { ThtRunner } from "../src/tht/tht-runner.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
const apps: ReturnType<typeof Fastify>[] = [];
|
||||
afterEach(async () => { await Promise.all(apps.splice(0).map(app => app.close())); });
|
||||
function setup(admin = true) {
|
||||
const app = Fastify(); apps.push(app);
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "curator", roles: [admin ? "admin" : "user"],
|
||||
permissions: admin ? ["evidence.manage"] : ["session.use"], isAdmin: admin };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const run = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ items: [], total: 0 }), stderr: "" }));
|
||||
const withPrincipal = vi.fn(() => ({ runWithRuntimeSnapshot: run }) as unknown as ThtRunner);
|
||||
const consolidateEvidence = vi.fn();
|
||||
const evidenceSources = vi.fn().mockResolvedValue({status: "succeeded"});
|
||||
evidenceRoutes(app, { runner: { withPrincipal }, registryRoot: "/data/registry", hostRegistryRoot: "/srv/Thoth workspaces",
|
||||
registry: { list: async () => [{ id: "sales", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/missing" }] },
|
||||
service: { consolidateEvidence, evidenceSources } });
|
||||
return { app, run, withPrincipal, principal, consolidateEvidence, evidenceSources };
|
||||
}
|
||||
test("browses complete local files with host paths, without a database runtime", async () => {
|
||||
const { app, run, withPrincipal, principal } = setup();
|
||||
const response = await app.inject("/workspaces/sales/evidence?kind=domain&concept=orders&page=2");
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(withPrincipal).toHaveBeenCalledWith(principal);
|
||||
const [argv, snapshot] = run.mock.calls[0] as unknown as [string[], string];
|
||||
expect(argv).toEqual(["evidence", "admin", "--workspace", "sales"]);
|
||||
expect(JSON.parse(snapshot)).toEqual({ root: "/data/registry/repo/sales", query: { kind: "domain", concept: "orders", page: 2 } });
|
||||
expect(response.json().location.workspace).toBe("/srv/Thoth workspaces/repo/sales");
|
||||
expect(response.json().location.git_commands[0]).toContain("git -C '/srv/Thoth workspaces/repo'");
|
||||
});
|
||||
|
||||
test("source actions require admin, bind the actor, and reject cross-workspace or arbitrary inputs", async () => {
|
||||
const denied = setup(false);
|
||||
expect((await denied.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: {action: "refresh"}})).statusCode).toBe(403);
|
||||
expect(denied.evidenceSources).not.toHaveBeenCalled();
|
||||
const allowed = setup();
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: {action: "refresh"}})).statusCode).toBe(200);
|
||||
expect(allowed.evidenceSources).toHaveBeenCalledWith({workspaceId: "sales", action: "refresh", actor: "curator"});
|
||||
const choice = {action: "decide", sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "keep"};
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: choice})).statusCode).toBe(200);
|
||||
expect(allowed.evidenceSources).toHaveBeenLastCalledWith({...choice, workspaceId: "sales", actor: "curator"});
|
||||
for (const payload of [{action: "refresh", url: "http://localhost"}, {...choice, actor: "forged"}, {...choice, revision: "../other"}]) {
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload})).statusCode).toBe(400);
|
||||
}
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/other/evidence/sources", payload: choice})).statusCode).toBe(404);
|
||||
expect(allowed.evidenceSources).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
test.each(["/", "/evidence:rule", "/consolidate"])("refuses non-admin access %s", async suffix => {
|
||||
const { app, run, consolidateEvidence } = setup(false);
|
||||
const response = await app.inject({ method: suffix === "/consolidate" ? "POST" : "GET", url: `/workspaces/sales/evidence${suffix === "/" ? "" : suffix}` });
|
||||
expect(response.statusCode).toBe(403); expect(run).not.toHaveBeenCalled(); expect(consolidateEvidence).not.toHaveBeenCalled();
|
||||
});
|
||||
test("rejects workspace escape, unknown workspace and unsupported filters", async () => {
|
||||
const { app, run } = setup();
|
||||
expect((await app.inject("/workspaces/foreign/evidence")).statusCode).toBe(404);
|
||||
expect((await app.inject("/workspaces/sales/evidence?root=/private")).statusCode).toBe(400);
|
||||
expect((await app.inject("/workspaces/sales/evidence?page_size=101")).statusCode).toBe(400);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
test("a missing unit is 404 and consolidation preserves the partial outcome", async () => {
|
||||
const { app, consolidateEvidence } = setup();
|
||||
expect((await app.inject("/workspaces/sales/evidence/evidence:missing")).statusCode).toBe(404);
|
||||
consolidateEvidence.mockResolvedValue({ status: "failed", warnings: ["Saved; retry indexing."] });
|
||||
const response = await app.inject({ method: "POST", url: "/workspaces/sales/evidence/consolidate" });
|
||||
expect(response.json()).toEqual({ status: "failed", warnings: ["Saved; retry indexing."] });
|
||||
});
|
||||
@@ -56,7 +56,7 @@ test("catalog listing translates upstream Pi IDs back to canonical model keys",
|
||||
session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
};
|
||||
try {
|
||||
@@ -75,6 +75,38 @@ test("catalog listing translates upstream Pi IDs back to canonical model keys",
|
||||
}
|
||||
});
|
||||
|
||||
test("catalog listing supplies the shared DeepSeek key without requiring Pi auth", async () => {
|
||||
const script = scriptWith([{ provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro" }]);
|
||||
const secret = join(path.dirname(script), "thothii.secrets");
|
||||
writeFileSync(secret, "DEEPSEEK_API_KEY=shared-key\nOPENAI_API_KEY=unrelated-key\n", { mode: 0o600 });
|
||||
const model: RuntimeModel = {
|
||||
id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
|
||||
label: "DeepSeek V4 Pro", upstreamModel: "deepseek-v4-pro",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: false },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
};
|
||||
try {
|
||||
const lister = createPiModelLister(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
...noManagedModels, modelCatalog, loadEnabledModels: enabled(model.id),
|
||||
spawnFn: (_command, _args, options) => {
|
||||
expect(options.env.DEEPSEEK_API_KEY).toBe("shared-key");
|
||||
expect(options.env).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(options.env).not.toHaveProperty("THT_SECRETS_FILE");
|
||||
return spawn("node", [FAKE, script], { env: options.env }) as any;
|
||||
},
|
||||
});
|
||||
await expect(lister()).resolves.toEqual([{
|
||||
provider: model.provider, id: model.model, name: model.label, reasoning: false,
|
||||
}]);
|
||||
} finally {
|
||||
rmSync(path.dirname(script), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
|
||||
const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import Fastify from "fastify";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { memoryRoutes } from "../src/routes/memory.js";
|
||||
import { DEFAULT_SEMANTIC_RUNTIME } from "../src/workspaces/runtime-renderer.js";
|
||||
import type { ThtRunner } from "../src/tht/tht-runner.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
const apps: ReturnType<typeof Fastify>[] = [];
|
||||
afterEach(async () => { await Promise.all(apps.splice(0).map(app => app.close())); });
|
||||
const id = "mem-11111111-1111-4111-8111-111111111111";
|
||||
const input = { family: "domain_clarification", subject: "Order", scope: "Sales" };
|
||||
|
||||
function setup(admin = true) {
|
||||
const app = Fastify(); apps.push(app);
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "operator", roles: [admin ? "admin" : "user"],
|
||||
permissions: admin ? ["memory.manage"] : ["session.use"], isAdmin: admin };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const run = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ items: [], total: 0 }), stderr: "" }));
|
||||
const bound = { runWithRuntimeSnapshot: run } as unknown as ThtRunner;
|
||||
const withPrincipal = vi.fn(() => bound);
|
||||
memoryRoutes(app, { runner: { withPrincipal }, runtime: DEFAULT_SEMANTIC_RUNTIME,
|
||||
registry: {
|
||||
list: async () => [{ id: "sales", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/missing" }],
|
||||
read: vi.fn().mockResolvedValue({ workspace: { workspace: { language: "it" } } }),
|
||||
} });
|
||||
return { app, run, withPrincipal, principal };
|
||||
}
|
||||
|
||||
test("list binds principal and workspace without requiring a DWH runtime", async () => {
|
||||
const { app, run, withPrincipal, principal } = setup();
|
||||
const response = await app.inject("/workspaces/sales/memory?q=Order&page=2&column=id");
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(withPrincipal).toHaveBeenCalledWith(principal);
|
||||
const [args, snapshot] = run.mock.calls[0] as unknown as [string[], string];
|
||||
expect(args).toEqual(["memory", "admin", "--workspace", "sales"]);
|
||||
expect(JSON.parse(snapshot)).toMatchObject({ action: "list", request: { q: "Order", page: 2, column: "id" } });
|
||||
expect(JSON.parse(snapshot).runtime.memoryLanguage).toBe("it");
|
||||
});
|
||||
|
||||
test.each([
|
||||
["GET", ""], ["POST", ""], ["GET", "/pending"], ["GET", `/${id}`],
|
||||
["PUT", `/${id}`], ["DELETE", `/${id}`], ["POST", `/${id}/retry`],
|
||||
] as const)("non-admin cannot %s %s", async (method, suffix) => {
|
||||
const { app, run } = setup(false);
|
||||
const response = await app.inject({ method, url: `/workspaces/sales/memory${suffix}`,
|
||||
...(method === "PUT" || method === "POST" && suffix === "" ? { payload: input } : {}) });
|
||||
expect(response.statusCode).toBe(403); expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("unknown workspace and invalid input do not invoke the harness", async () => {
|
||||
const { app, run } = setup();
|
||||
expect((await app.inject("/workspaces/missing/memory")).statusCode).toBe(404);
|
||||
expect((await app.inject("/workspaces/sales/memory?page_size=101")).statusCode).toBe(400);
|
||||
expect((await app.inject({ method: "POST", url: "/workspaces/sales/memory",
|
||||
payload: { ...input, workspace_id: "foreign" } })).statusCode).toBe(400);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("create preserves saved-but-unindexed outcome, update carries complete related changes", async () => {
|
||||
const { app, run } = setup();
|
||||
run.mockResolvedValue({ code: 0, stdout: JSON.stringify({ id, saved: true, indexed: false }), stderr: "" });
|
||||
const result = await app.inject({ method: "POST", url: "/workspaces/sales/memory", payload: input });
|
||||
expect(result.statusCode).toBe(201); expect(result.json()).toEqual({ id, saved: true, indexed: false });
|
||||
await app.inject({ method: "PUT", url: `/workspaces/sales/memory/${id}`, payload: {
|
||||
...input, links: [{ target_id: id, meaning: "Related" }], dependencies: [{ database: "dwh" }],
|
||||
} });
|
||||
const [, snapshot] = run.mock.calls[1] as unknown as [string[], string];
|
||||
expect(JSON.parse(snapshot)).toMatchObject({ action: "update", request: { id, card: {
|
||||
links: [{ target_id: id, meaning: "Related" }], dependencies: [{ database: "dwh" }],
|
||||
} } });
|
||||
});
|
||||
|
||||
test.each([["memory_not_found", 404], ["memory_conflict", 409], ["memory_unavailable", 503]])(
|
||||
"maps %s without leaking stderr", async (code, status) => {
|
||||
const { app, run } = setup();
|
||||
run.mockResolvedValue({ code: 1, stdout: JSON.stringify({ code, message: "sensitive detail" }), stderr: "secret" });
|
||||
const result = await app.inject("/workspaces/sales/memory");
|
||||
expect(result.statusCode).toBe(status); expect(result.json().code).toBe(code);
|
||||
expect(result.body).not.toMatch(/secret|sensitive/);
|
||||
},
|
||||
);
|
||||
@@ -20,9 +20,8 @@ function runtimeCatalog(overrides: Record<string, unknown> = {}, secrets = "OPEN
|
||||
const catalogFile = join(root, "catalog.json");
|
||||
const secretsFile = join(root, "thothii.secrets");
|
||||
const catalog = {
|
||||
schemaVersion: 1,
|
||||
defaultSession: "zai/glm-5.3",
|
||||
defaultMetadataGeneration: "zai/glm-5.3",
|
||||
schemaVersion: 2,
|
||||
defaultInteraction: "zai/glm-5.3",
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
models: [
|
||||
{
|
||||
@@ -55,8 +54,8 @@ test("loads session default and safe metadata choices from the normalized runtim
|
||||
const runtime = loadRuntimeModelCatalog(catalogFile);
|
||||
const metadata = loadMetadataGenerationModels({ catalogFile, secretsFile });
|
||||
|
||||
expect(runtime.defaultSession).toBe("zai/glm-5.3");
|
||||
expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(true);
|
||||
expect(runtime.defaultInteraction).toBe("zai/glm-5.3");
|
||||
expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(false);
|
||||
expect(metadata.catalog()).toEqual({
|
||||
models: [{ id: "zai/glm-5.3", label: "GLM 5.3" }],
|
||||
default: "zai/glm-5.3",
|
||||
@@ -69,14 +68,65 @@ test("loads session default and safe metadata choices from the normalized runtim
|
||||
expect(() => metadata.resolve("zai/missing")).toThrow(MetadataGenerationModelUnavailableError);
|
||||
});
|
||||
|
||||
test("DeepSeek uses one identity and bundle credential for native Pi and LiteLLM", () => {
|
||||
const models = ["deepseek-v4-pro", "deepseek-v4-flash"].map((model) => ({
|
||||
id: `deepseek/${model}`, provider: "deepseek", model, label: model, upstreamModel: model,
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" }, metadataAdapter: { litellmProvider: "deepseek" },
|
||||
session: { reasoning: false }, metadataGeneration: { disableThinking: false },
|
||||
}));
|
||||
const files = runtimeCatalog({ defaultInteraction: models[0].id, models }, "DEEPSEEK_API_KEY=shared-key\n");
|
||||
const runtime = loadRuntimeModelCatalog(files.catalogFile);
|
||||
const metadata = loadMetadataGenerationModels(files);
|
||||
expect(runtime.sessionModels().map((model) => model.id)).toEqual(metadata.catalog().models.map((model) => model.id));
|
||||
expect(metadata.catalog().default).toBe(runtime.defaultInteraction);
|
||||
for (const model of models) {
|
||||
expect(metadata.resolve(model.id)).toMatchObject({
|
||||
id: model.id, provider: "deepseek", model: model.model,
|
||||
apiKeyEnv: "DEEPSEEK_API_KEY", apiKey: "shared-key",
|
||||
});
|
||||
}
|
||||
expect(() => metadata.resolve("deepseek-metadata/deepseek-v4-pro"))
|
||||
.toThrow(MetadataGenerationModelUnavailableError);
|
||||
});
|
||||
|
||||
test("returns empty catalogs when no runtime projection is configured", () => {
|
||||
expect(loadRuntimeModelCatalog().defaultSession).toBeNull();
|
||||
expect(loadRuntimeModelCatalog().defaultInteraction).toBeNull();
|
||||
expect(loadMetadataGenerationModels({}).catalog()).toEqual({ models: [], default: null });
|
||||
});
|
||||
|
||||
test.each([
|
||||
["qwen-chat-template", true, true],
|
||||
["qwen", true, true],
|
||||
["qwen-typo", true, false],
|
||||
["qwen-chat-template", false, false],
|
||||
])("validates Qwen thinking format %s with reasoning=%s", (thinkingFormat, reasoning, valid) => {
|
||||
const { catalogFile } = runtimeCatalog({
|
||||
defaultInteraction: "local/qwen",
|
||||
models: [{
|
||||
id: "local/qwen", provider: "local", model: "qwen", label: "Qwen",
|
||||
upstreamModel: "qwen", endpoint: { baseUrl: "http://localhost:8000/v1" },
|
||||
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
|
||||
session: {
|
||||
reasoning, contextWindow: 32768, maxTokens: 8192,
|
||||
compatibility: {
|
||||
supportsDeveloperRole: false, supportsReasoningEffort: false,
|
||||
supportsStore: false, maxTokensField: "max_tokens", thinkingFormat,
|
||||
},
|
||||
},
|
||||
}],
|
||||
});
|
||||
if (valid) {
|
||||
expect(loadRuntimeModelCatalog(catalogFile).sessionModels()[0].session?.compatibility)
|
||||
.toMatchObject({ thinkingFormat });
|
||||
} else {
|
||||
expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid");
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects a drifted default and an unprotected projection", () => {
|
||||
const drifted = runtimeCatalog({ defaultSession: "zai/missing" });
|
||||
expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("session default is invalid");
|
||||
const drifted = runtimeCatalog({ defaultInteraction: "zai/missing" });
|
||||
expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("interaction default is invalid");
|
||||
|
||||
const unprotected = runtimeCatalog();
|
||||
chmodSync(unprotected.catalogFile, 0o666);
|
||||
@@ -85,7 +135,7 @@ test("rejects a drifted default and an unprotected projection", () => {
|
||||
|
||||
test("rejects authentication semantics that cannot come from the installation catalog", () => {
|
||||
const invalid = runtimeCatalog({
|
||||
defaultMetadataGeneration: undefined,
|
||||
defaultInteraction: undefined,
|
||||
models: [{
|
||||
id: "zai/glm-5.3",
|
||||
provider: "zai",
|
||||
@@ -112,3 +162,13 @@ test("splits canonical session identities without provider aliases", () => {
|
||||
expect(splitCanonicalModelId("zai/glm-5.3")).toEqual({ provider: "zai", model: "glm-5.3" });
|
||||
expect(() => splitCanonicalModelId("glm-5.3")).toThrow("model identity is invalid");
|
||||
});
|
||||
|
||||
test("rejects a default supported by only one configured use", () => {
|
||||
const { catalogFile } = runtimeCatalog({ defaultInteraction: "deepseek/deepseek-v4-pro" });
|
||||
expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("interaction default is invalid");
|
||||
});
|
||||
|
||||
test("requires regenerated runtime schema v2 rather than interpreting two legacy defaults", () => {
|
||||
const { catalogFile } = runtimeCatalog({ schemaVersion: 1, defaultSession: "zai/glm-5.3" });
|
||||
expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { beforeEach, expect, test, vi } from "vitest";
|
||||
import type { AppConfig } from "../src/config.js";
|
||||
|
||||
const fakes = vi.hoisted(() => ({
|
||||
catalogRepository: { close: vi.fn(async () => {}) },
|
||||
createCatalogRepository: vi.fn(),
|
||||
runnerConfig: undefined as Record<string, unknown> | undefined,
|
||||
acquireWorkspaceRuntime: vi.fn(),
|
||||
releaseWorkspaceRuntime: vi.fn(),
|
||||
run: vi.fn(async () => ({
|
||||
code: 0,
|
||||
stdout: JSON.stringify({ ok: true }),
|
||||
stderr: "",
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../src/catalog/repository.js", () => ({
|
||||
createCatalogRepository: fakes.createCatalogRepository,
|
||||
}));
|
||||
|
||||
vi.mock("../src/tht/tht-runner.js", () => ({
|
||||
ThtRunner: class {
|
||||
constructor(config: Record<string, unknown>) {
|
||||
fakes.runnerConfig = config;
|
||||
}
|
||||
|
||||
run = fakes.run;
|
||||
|
||||
acquireWorkspaceRuntime = fakes.acquireWorkspaceRuntime;
|
||||
|
||||
withPrincipal() {
|
||||
return this;
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../src/workspaces/registry.js", () => ({
|
||||
WorkspaceRegistry: class {
|
||||
async listRetainedSnapshots() {
|
||||
return [{ snapshotPath: "/data/workspace-registry/snapshots/revision/workspace.yaml" }];
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../src/workspaces/secret-store.js", () => ({
|
||||
WorkspaceSecretStore: class {},
|
||||
}));
|
||||
|
||||
import { runOperatorAction } from "../src/operator-command.js";
|
||||
|
||||
const config = {
|
||||
catalogDatabase: { host: "catalog-db" },
|
||||
workspaceSecretStoreRoot: "/data/workspace-secrets",
|
||||
workspaceSecretRuntimeRoot: "/tmp/workspace-secrets",
|
||||
workspaceRegistry: {
|
||||
installationId: "test",
|
||||
root: "/data/workspace-registry",
|
||||
secretRoots: ["/run/secrets"],
|
||||
},
|
||||
thtBin: "/opt/venv/bin/tht",
|
||||
harnessDir: "/app/harness",
|
||||
dataRoot: "/data",
|
||||
internalQdrantUrl: "http://qdrant:6333",
|
||||
internalEmbeddingUrl: "http://embedding:11434",
|
||||
internalEmbeddingModel: "qwen3-embedding:0.6b",
|
||||
internalEmbeddingDimensions: 1024,
|
||||
} as AppConfig;
|
||||
|
||||
beforeEach(() => {
|
||||
fakes.catalogRepository.close.mockClear();
|
||||
fakes.createCatalogRepository.mockReset();
|
||||
fakes.createCatalogRepository.mockReturnValue(fakes.catalogRepository);
|
||||
fakes.run.mockClear();
|
||||
fakes.acquireWorkspaceRuntime.mockReset();
|
||||
fakes.acquireWorkspaceRuntime.mockResolvedValue({
|
||||
path: "/data/workspace-registry/snapshots/runtime/workspace.yaml",
|
||||
release: fakes.releaseWorkspaceRuntime,
|
||||
});
|
||||
fakes.releaseWorkspaceRuntime.mockClear();
|
||||
fakes.runnerConfig = undefined;
|
||||
});
|
||||
|
||||
test("workflow doctor gives schema-v4 runtime rendering a live Catalog repository", async () => {
|
||||
await expect(runOperatorAction("workflow-doctor", config)).resolves.toEqual({
|
||||
ready: true,
|
||||
workspaces: 1,
|
||||
});
|
||||
|
||||
expect(fakes.createCatalogRepository).toHaveBeenCalledWith(config.catalogDatabase);
|
||||
expect(fakes.runnerConfig?.catalogRepository).toBe(fakes.catalogRepository);
|
||||
expect(fakes.acquireWorkspaceRuntime).toHaveBeenCalledWith(
|
||||
"/data/workspace-registry/snapshots/revision/workspace.yaml",
|
||||
);
|
||||
expect(fakes.run).toHaveBeenCalledWith(
|
||||
["doctor", "--json"],
|
||||
"/data/workspace-registry/snapshots/runtime/workspace.yaml",
|
||||
);
|
||||
expect(fakes.releaseWorkspaceRuntime).toHaveBeenCalledOnce();
|
||||
expect(fakes.catalogRepository.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, test, vi } from "vitest";
|
||||
@@ -7,7 +7,34 @@ import {
|
||||
createPiManagement,
|
||||
type PiExecFile,
|
||||
} from "../src/pi/management.js";
|
||||
import type { RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
|
||||
import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
|
||||
|
||||
test.each([false, true])("catalog credential status ignores legacy Pi auth, missing bundle key: %s", async (missingKey) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "tht-catalog-status-"));
|
||||
writeFileSync(join(root, "auth.json"), JSON.stringify({ deepseek: { type: "api_key", key: "stale-key" } }), { mode: 0o600 });
|
||||
const secret = join(root, "thothii.secrets");
|
||||
writeFileSync(secret, missingKey ? "THT_MODEL_API_KEY=legacy-key\n" : "DEEPSEEK_API_KEY=shared-key\n", { mode: 0o600 });
|
||||
vi.stubEnv("PI_CODING_AGENT_DIR", root);
|
||||
const model: RuntimeModel = {
|
||||
id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
|
||||
label: "DeepSeek", upstreamModel: "deepseek-v4-pro",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: false },
|
||||
};
|
||||
try {
|
||||
const service = createPiManagement(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
modelCatalog: {
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
},
|
||||
execute: successfulExec([]), readSettings: () => ({ thinking: "medium" }),
|
||||
});
|
||||
expect((await service.status()).credentials).toBe(missingKey ? "missing" : "present");
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-management-")), "settings.json")) {
|
||||
return loadConfig({
|
||||
@@ -15,12 +42,12 @@ function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-manage
|
||||
SETTINGS_FILE: settingsFile,
|
||||
PI_BIN: "/usr/local/bin/pi",
|
||||
PI_MANAGEMENT_TIMEOUT_MS: "750",
|
||||
THT_HOST_PLATFORM: "linux",
|
||||
});
|
||||
}
|
||||
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: "zai/glm-5.2",
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: "zai/glm-5.2",
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
sessionModels: () => [],
|
||||
metadataModels: () => [],
|
||||
@@ -34,6 +61,16 @@ function successfulExec(calls: Array<{ command: string; args: string[]; timeout:
|
||||
};
|
||||
}
|
||||
|
||||
test.each([
|
||||
["linux", "linux"], ["darwin", "macos"], ["windows", "windows"],
|
||||
])("reports installation host %s independently of the backend container OS", async (host, expected) => {
|
||||
const service = createPiManagement(loadConfig({ THT_HOST_PLATFORM: host }), {
|
||||
modelCatalog, execute: successfulExec([]), readSettings: () => ({ thinking: "medium" }),
|
||||
credentialStatus: () => "missing",
|
||||
});
|
||||
expect((await service.status()).hostPlatform).toBe(expected);
|
||||
});
|
||||
|
||||
// Catches a Pi executable that emits unexpected text or is invoked through a shell, which could
|
||||
// turn a version display into a command-injection or information-disclosure surface.
|
||||
test("status parses only a Pi version from a fixed execFile argument array", async () => {
|
||||
@@ -47,6 +84,7 @@ test("status parses only a Pi version from a fixed execFile argument array", asy
|
||||
});
|
||||
|
||||
await expect(service.status()).resolves.toEqual({
|
||||
hostPlatform: "linux",
|
||||
version: "0.80.3",
|
||||
ready: true,
|
||||
credentials: "missing",
|
||||
@@ -78,6 +116,7 @@ test.each(["present", "missing"] as const)(
|
||||
|
||||
const status = await service.status();
|
||||
expect(status).toEqual({
|
||||
hostPlatform: "linux",
|
||||
version: "0.80.3",
|
||||
ready: true,
|
||||
credentials,
|
||||
|
||||
@@ -190,6 +190,29 @@ test("Pi receives the leased workspace runtime config and releases it on direct
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("Pi language launch hint comes from the session options and never ambient environment", () => {
|
||||
const previous = process.env.THT_INTERACTION_LANGUAGE;
|
||||
process.env.THT_INTERACTION_LANGUAGE = "it";
|
||||
const environments: NodeJS.ProcessEnv[] = [];
|
||||
const mgr = new PiProcessManager(loadConfig({}), {
|
||||
spawnFn: (_command, _args, options) => {
|
||||
environments.push(options.env);
|
||||
return recordingChild() as any;
|
||||
},
|
||||
});
|
||||
try {
|
||||
mgr.createFor("explicit-language", { interactionLanguage: "en" });
|
||||
mgr.teardown("explicit-language");
|
||||
mgr.createFor("manifest-resolved-in-gate");
|
||||
mgr.teardown("manifest-resolved-in-gate");
|
||||
expect(environments[0].THT_INTERACTION_LANGUAGE).toBe("en");
|
||||
expect(environments[1]).not.toHaveProperty("THT_INTERACTION_LANGUAGE");
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.THT_INTERACTION_LANGUAGE;
|
||||
else process.env.THT_INTERACTION_LANGUAGE = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test("a close-only child event releases its temporary Pi agent snapshot", () => {
|
||||
const child = recordingChild();
|
||||
let snapshotDir: string | undefined;
|
||||
@@ -642,27 +665,33 @@ test("session Pi spawn reads the single secret bundle and scrubs its path", asyn
|
||||
}
|
||||
});
|
||||
|
||||
test("session Pi spawn resolves the selected catalog credential from the secret bundle", () => {
|
||||
test.each([false, true])("session Pi uses the catalog bundle despite stale Pi auth: %s", (missingKey) => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "thothii-catalog-credential-"));
|
||||
const agentDir = path.join(root, "agent");
|
||||
mkdirSync(agentDir, { mode: 0o700 });
|
||||
writeFileSync(path.join(agentDir, "auth.json"), "{}\n", { mode: 0o600 });
|
||||
const originalAuth = JSON.stringify({
|
||||
deepseek: { type: "api_key", key: "stale-key" },
|
||||
anthropic: { type: "api_key", key: "unrelated-key" },
|
||||
});
|
||||
writeFileSync(path.join(agentDir, "auth.json"), originalAuth, { mode: 0o600 });
|
||||
writeFileSync(path.join(agentDir, "models.json"), '{"providers":{}}\n', { mode: 0o600 });
|
||||
const secret = path.join(root, "thothii.secrets");
|
||||
writeFileSync(secret, "ZAI_API_KEY=catalog-secret\nTHT_MODEL_API_KEY=legacy-secret\n", { mode: 0o600 });
|
||||
writeFileSync(secret, missingKey ? "THT_MODEL_API_KEY=legacy-secret\n"
|
||||
: "DEEPSEEK_API_KEY=catalog-secret\nTHT_MODEL_API_KEY=legacy-secret\n", { mode: 0o600 });
|
||||
const legacyKey = path.join(root, "legacy-key");
|
||||
writeFileSync(legacyKey, "legacy-file-key", { mode: 0o600 });
|
||||
const model: RuntimeModel = {
|
||||
id: "openai/test-model",
|
||||
provider: "openai",
|
||||
model: "test-model",
|
||||
label: "Test model",
|
||||
upstreamModel: "test-model",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "ZAI_API_KEY" },
|
||||
id: "deepseek/deepseek-v4-pro",
|
||||
provider: "deepseek",
|
||||
model: "deepseek-v4-pro",
|
||||
label: "DeepSeek V4 Pro",
|
||||
upstreamModel: "deepseek-v4-pro",
|
||||
authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
|
||||
sessionAdapter: { mode: "pi_builtin" },
|
||||
session: { reasoning: false },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: model.id,
|
||||
embedding: null,
|
||||
sessionModels: () => [model],
|
||||
metadataModels: () => [],
|
||||
@@ -672,17 +701,26 @@ test("session Pi spawn resolves the selected catalog credential from the secret
|
||||
const child = recordingChild();
|
||||
child.stderr.resume = () => {};
|
||||
vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
|
||||
const mgr = new PiProcessManager(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
const mgr = new PiProcessManager(loadConfig({ THT_SECRETS_FILE: secret, THT_MODEL_API_KEY_FILE: legacyKey }), {
|
||||
modelCatalog,
|
||||
authProviders: () => new Set(),
|
||||
authProviders: () => new Set(["deepseek"]),
|
||||
spawnFn: (...args: any[]) => { calls.push(args); return child as any; },
|
||||
});
|
||||
try {
|
||||
mgr.createFor("catalog-credential", { provider: "openai", model: "test-model" });
|
||||
expect(calls[0][2].env.ZAI_API_KEY).toBe("catalog-secret");
|
||||
const create = () => mgr.createFor("catalog-credential", { provider: model.provider, model: model.model });
|
||||
if (missingKey) {
|
||||
expect(create).toThrow("model provider credential is unavailable");
|
||||
expect(calls).toHaveLength(0);
|
||||
return;
|
||||
}
|
||||
create();
|
||||
expect(calls[0][2].env.DEEPSEEK_API_KEY).toBe("catalog-secret");
|
||||
expect(calls[0][2].env).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(calls[0][2].env).not.toHaveProperty("THT_MODEL_API_KEY");
|
||||
expect(JSON.parse(readFileSync(path.join(calls[0][2].env.PI_CODING_AGENT_DIR, "auth.json"), "utf8")))
|
||||
.toEqual({ anthropic: { type: "api_key", key: "unrelated-key" } });
|
||||
} finally {
|
||||
expect(readFileSync(path.join(agentDir, "auth.json"), "utf8")).toBe(originalAuth);
|
||||
mgr.teardown("catalog-credential");
|
||||
vi.unstubAllEnvs();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
@@ -750,7 +788,7 @@ test("set_model translates a canonical catalog key to its upstream Pi model ID",
|
||||
session: { reasoning: false, contextWindow: 32768, maxTokens: 8192 },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
|
||||
defaultInteraction: model.id, embedding: null,
|
||||
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||
};
|
||||
const mgr = new PiProcessManager(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
|
||||
|
||||
@@ -61,20 +61,22 @@ test("provider smoke resolves the selected catalog credential from the secret bu
|
||||
session: { reasoning: false },
|
||||
};
|
||||
const modelCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: model.id,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: model.id,
|
||||
embedding: null,
|
||||
sessionModels: () => [model],
|
||||
metadataModels: () => [],
|
||||
hasSession: (id) => id === model.id,
|
||||
};
|
||||
let spawnEnv: NodeJS.ProcessEnv | undefined;
|
||||
const readAuthStore = vi.fn(() => JSON.stringify({ openai: { type: "api_key", key: "stale-key" } }));
|
||||
const smoke = createPiProviderSmoke(loadConfig({ THT_SECRETS_FILE: secret }), {
|
||||
modelCatalog,
|
||||
authProviders: () => new Set(),
|
||||
authProviders: () => new Set(["openai"]),
|
||||
readAuthStore,
|
||||
readModelsStore: () => undefined,
|
||||
spawnFn: (_command, _args, options) => {
|
||||
spawnEnv = options.env;
|
||||
expect(existsSync(join(options.env.PI_CODING_AGENT_DIR!, "auth.json"))).toBe(false);
|
||||
return successfulProviderChild();
|
||||
},
|
||||
});
|
||||
@@ -85,6 +87,7 @@ test("provider smoke resolves the selected catalog credential from the secret bu
|
||||
expect(spawnEnv?.ZAI_API_KEY).toBe("catalog-secret");
|
||||
expect(spawnEnv).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(spawnEnv).not.toHaveProperty("THT_MODEL_API_KEY");
|
||||
expect(readAuthStore).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -306,7 +309,7 @@ test("provider smoke makes one configured request from an isolated no-capability
|
||||
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true },
|
||||
};
|
||||
const smokeCatalog: RuntimeModelCatalog = {
|
||||
defaultSession: smokeModel.id, defaultMetadataGeneration: null, embedding: null,
|
||||
defaultInteraction: smokeModel.id, embedding: null,
|
||||
sessionModels: () => [smokeModel], metadataModels: () => [],
|
||||
hasSession: (id) => id === smokeModel.id,
|
||||
};
|
||||
|
||||
@@ -19,11 +19,6 @@ const validYaml = `workspace:
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
async function git(cwd: string, args: string[]): Promise<string> {
|
||||
|
||||
@@ -24,11 +24,6 @@ const descriptor = `workspace:
|
||||
id: research
|
||||
name: Research
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: filesystem
|
||||
|
||||
@@ -12,6 +12,7 @@ import { PiProcessManager } from "../src/pi/pi-process-manager.js";
|
||||
import { validateDeclarativePiConfig } from "../src/pi/managed-config.js";
|
||||
import Fastify from "fastify";
|
||||
import { sessionRoutes } from "../src/routes/sessions.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
|
||||
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
|
||||
const SCRIPT = path.resolve("../harness/tests/fake_pi/scripts/f1_disambiguation.json");
|
||||
@@ -26,10 +27,9 @@ function operationalWorkspace(id = "default") {
|
||||
} as const;
|
||||
}
|
||||
|
||||
function sessionCatalog(defaultSession = "zai/glm-5.2", available = [defaultSession]) {
|
||||
function sessionCatalog(defaultInteraction = "zai/glm-5.2", available = [defaultInteraction]) {
|
||||
return {
|
||||
defaultSession,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction,
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
sessionModels: () => [],
|
||||
metadataModels: () => [],
|
||||
@@ -53,7 +53,11 @@ const defaultWorkspaceRegistry = {
|
||||
|
||||
function buildApp(config: Parameters<typeof buildRealApp>[0], deps: Record<string, unknown> = {}) {
|
||||
const thtRunner = deps.thtRunner
|
||||
? { qdrantEnsure: async () => ({ ok: true }), ...(deps.thtRunner as object) }
|
||||
? {
|
||||
qdrantEnsure: async () => ({ ok: true }),
|
||||
ensureInteractionLanguage: async () => ({ interaction_language: "en" }),
|
||||
...(deps.thtRunner as object),
|
||||
}
|
||||
: undefined;
|
||||
return buildRealApp(config, {
|
||||
workspaceRuntimeSupport: () => true,
|
||||
@@ -88,6 +92,156 @@ const aliceHeaders = {
|
||||
"x-thoth-is-admin": "0",
|
||||
};
|
||||
|
||||
test.each([undefined, null, "", "en--US", "en_US", "en\nIGNORE", "en<script>", 42])(
|
||||
"new sessions reject invalid interaction language %s before persistence", async (interactionLanguage) => {
|
||||
const app = mutApp({ sessionNew: async () => { throw new Error("must not create"); } });
|
||||
try {
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "Pazienti", interactionLanguage },
|
||||
});
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(response.json()).toMatchObject({ code: "invalid_interaction_language" });
|
||||
} finally { await app.close(); }
|
||||
},
|
||||
);
|
||||
|
||||
test.each([["en", "en"], ["fr-FR", "fr-FR"], ["FR-fr", "fr-FR"]])(
|
||||
"new session forwards canonical interaction language %s without changing the question", async (language, canonical) => {
|
||||
let created: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionNew: async (options: any) => { created = options; return { id: "language" }; },
|
||||
searchPack: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined, createFor: () => ({ bridge: { onClientEvent: () => {} } }),
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "default" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "Pazienti", interactionLanguage: language },
|
||||
});
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(created).toMatchObject({ question: "Pazienti", interactionLanguage: canonical });
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume rejects browser interaction language overrides", async () => {
|
||||
const app = mutApp({ sessionShow: async () => ({ status: "open", interaction_language: "it" }) });
|
||||
try {
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions/s1/resume", payload: { interactionLanguage: "en" },
|
||||
});
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(response.json()).toMatchObject({ code: "interaction_language_pinned" });
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("new session starts Pi with the question language persisted by the harness", async () => {
|
||||
let runtime: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionNew: async () => ({ id: "english-question", interaction_language: "en" }),
|
||||
searchPack: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: (_id: string, options: any) => {
|
||||
runtime = options;
|
||||
return { bridge: { onClientEvent: () => {} } };
|
||||
},
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "default" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: {
|
||||
question: "How many patients were admitted last year?", interactionLanguage: "it",
|
||||
} });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(runtime.interactionLanguage).toBe("en");
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume pins legacy interaction language using the resolved harness workspace", async () => {
|
||||
let pinnedWorkspace: string | undefined;
|
||||
let runtime: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ id: "legacy", status: "closed" }),
|
||||
ensureInteractionLanguage: async (_id: string, workspace: string) => {
|
||||
pinnedWorkspace = workspace;
|
||||
return { interaction_language: "it" };
|
||||
},
|
||||
reopenSession: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: (_id: string, options: any) => {
|
||||
runtime = options;
|
||||
return { bridge: { onClientEvent: () => {} } };
|
||||
},
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "other-browser-workspace" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/legacy/resume" });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(pinnedWorkspace).toContain("/default.yaml");
|
||||
expect(runtime).toMatchObject({ interactionLanguage: "it", mode: "resume" });
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume retains persisted interaction language despite different browser settings", async () => {
|
||||
let options: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ id: "saved", status: "closed", interaction_language: "it" }),
|
||||
ensureInteractionLanguage: async () => { throw new Error("language is already pinned"); },
|
||||
reopenSession: async () => {},
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: (_id: string, value: any) => {
|
||||
options = value;
|
||||
return { bridge: { onClientEvent: () => {} } };
|
||||
},
|
||||
configure: async () => {}, start: () => {},
|
||||
},
|
||||
getSettings: () => ({ workspace: "english-workspace", locale: "en" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/saved/resume" });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(options.interactionLanguage).toBe("it");
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("resume cannot start Pi if legacy interaction language cannot be persisted", async () => {
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ id: "legacy", status: "closed" }),
|
||||
ensureInteractionLanguage: async () => { throw new Error("private storage details"); },
|
||||
reopenSession: async () => { throw new Error("must not reopen"); },
|
||||
},
|
||||
readiness: { ensure: async () => ({ ok: true }) },
|
||||
mgr: { createFor: () => { throw new Error("must not spawn"); } },
|
||||
getSettings: () => ({ workspace: "default" }),
|
||||
});
|
||||
try {
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/legacy/resume" });
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.body).not.toContain("private storage details");
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("upstream requests without a principal fail before a Pi runtime can be created", async () => {
|
||||
let created = false;
|
||||
const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), {
|
||||
@@ -95,7 +249,7 @@ test("upstream requests without a principal fail before a Pi runtime can be crea
|
||||
thtRunner: {} as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(401);
|
||||
expect(created).toBe(false);
|
||||
@@ -133,7 +287,7 @@ test("maintenance rejects new and resumed session admission without interrupting
|
||||
});
|
||||
|
||||
const create = await app.inject({
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { question: "q" },
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
const resume = await app.inject({ method: "POST", url: "/sessions/open/resume", headers: aliceHeaders });
|
||||
|
||||
@@ -154,7 +308,7 @@ test("a durable maintenance marker initializes admission closed after backend re
|
||||
AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness", THT_MAINTENANCE_FILE: marker,
|
||||
}), { thtRunner: {} as any });
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { question: "q" },
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders, payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.json()).toMatchObject({ code: "maintenance" });
|
||||
@@ -363,6 +517,65 @@ test("retention scans a removed workspace's retained snapshot", async () => {
|
||||
expect(response.json()).toEqual([expect.objectContaining({ id: "resumable" })]);
|
||||
});
|
||||
|
||||
test("active sessions remain available when retained snapshot discovery is unreadable", async () => {
|
||||
const activeSnapshot = "/registry/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/active.yaml";
|
||||
const retainedFailure = "invalid retained snapshot /registry/snapshots/secret/legacy.yaml";
|
||||
const reconcileSnapshotRetention = vi.fn(async () => {});
|
||||
const subscribed = vi.fn();
|
||||
const warning = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const manifest = {
|
||||
id: "live-session", status: "open", archived: false,
|
||||
workspace_id: "active", workspace_revision: "a".repeat(40),
|
||||
};
|
||||
const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
withPrincipal: () => ({
|
||||
sessionList: async (snapshotPath: string) => {
|
||||
expect(snapshotPath).toBe(activeSnapshot);
|
||||
return [manifest];
|
||||
},
|
||||
sessionShow: async (id: string, snapshotPath?: string) => {
|
||||
if (id === manifest.id && snapshotPath === activeSnapshot) return manifest;
|
||||
throw new Error("session not found");
|
||||
},
|
||||
}),
|
||||
} as any,
|
||||
mgr: { get: () => undefined } as any,
|
||||
hub: {
|
||||
subscribe: (_id: string, _send: unknown, options: { close?: () => void }) => {
|
||||
subscribed();
|
||||
options.close?.();
|
||||
return () => {};
|
||||
},
|
||||
} as any,
|
||||
workspaceRegistry: {
|
||||
listRetainedSnapshots: async () => { throw new Error(retainedFailure); },
|
||||
list: async () => [{
|
||||
id: "active", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: activeSnapshot,
|
||||
}],
|
||||
reconcileSnapshotRetention,
|
||||
} as any,
|
||||
});
|
||||
|
||||
try {
|
||||
const list = await app.inject({ method: "GET", url: "/sessions", headers: aliceHeaders });
|
||||
const detail = await app.inject({ method: "GET", url: `/sessions/${manifest.id}`, headers: aliceHeaders });
|
||||
const events = await app.inject({ method: "GET", url: `/sessions/${manifest.id}/events`, headers: aliceHeaders });
|
||||
|
||||
expect(list.statusCode).toBe(200);
|
||||
expect(list.json()).toEqual([expect.objectContaining({ id: manifest.id, active: false })]);
|
||||
expect(detail.statusCode).toBe(200);
|
||||
expect(detail.json()).toMatchObject(manifest);
|
||||
expect(events.statusCode).toBe(200);
|
||||
expect(subscribed).toHaveBeenCalledOnce();
|
||||
expect(reconcileSnapshotRetention).not.toHaveBeenCalled();
|
||||
expect(list.body + detail.body + events.body).not.toContain(retainedFailure);
|
||||
} finally {
|
||||
warning.mockRestore();
|
||||
await app.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("the single local installation listing reconciles its resumable workspace pins", async () => {
|
||||
const retained = vi.fn(async () => {});
|
||||
const retainedRevision = "d".repeat(40);
|
||||
@@ -433,7 +646,7 @@ test("new sessions are created through the authenticated principal, not a client
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", headers: aliceHeaders,
|
||||
payload: { question: "q", owner: "mallory" },
|
||||
payload: { interactionLanguage: "en", question: "q", owner: "mallory" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
@@ -450,7 +663,7 @@ test("new sessions reject the client legacy workspace field unless local legacy
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "q", workspace: "legacy" },
|
||||
method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q", workspace: "legacy" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
@@ -470,7 +683,7 @@ test("explicit local legacy mode permits the unpinned client workspace request",
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "q", workspace: "legacy" },
|
||||
method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q", workspace: "legacy" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
@@ -508,7 +721,7 @@ test("creates a session from the active immutable workspace revision", async ()
|
||||
|
||||
await app.inject({
|
||||
method: "POST", url: "/sessions",
|
||||
payload: { question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" },
|
||||
payload: { interactionLanguage: "en", question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" },
|
||||
});
|
||||
|
||||
expect(sessionNew).toHaveBeenCalledWith(expect.objectContaining({
|
||||
@@ -517,7 +730,7 @@ test("creates a session from the active immutable workspace revision", async ()
|
||||
}));
|
||||
});
|
||||
|
||||
test("hands one effective relationship snapshot to both retrieval and Pi", async () => {
|
||||
test("hands one Catalog-backed runtime to both retrieval and Pi", async () => {
|
||||
const effective = JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
workspaceId: "default",
|
||||
@@ -552,14 +765,13 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "Which users placed orders?", workspaceId: "default" },
|
||||
payload: { interactionLanguage: "en", question: "Which users placed orders?", workspaceId: "default" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(render).toHaveBeenCalledWith("default");
|
||||
expect(render).not.toHaveBeenCalled();
|
||||
expect(acquireWorkspaceRuntime).toHaveBeenCalledWith(
|
||||
expect.stringContaining("/default.yaml"),
|
||||
effective,
|
||||
);
|
||||
expect(createFor).toHaveBeenCalledWith(
|
||||
"effective-map",
|
||||
@@ -574,27 +786,103 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects an SSH-only workspace before persisting or starting a session", async () => {
|
||||
test("refuses core admission when the workspace preprocessing fingerprint is stale", async () => {
|
||||
const sessionNew = vi.fn();
|
||||
const workspaceInputFingerprint = vi.fn(async () => "sha256:current");
|
||||
const revision = {
|
||||
id: "default",
|
||||
commit: "a".repeat(40),
|
||||
blob: "b".repeat(40),
|
||||
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/default.yaml`,
|
||||
};
|
||||
const catalogRepository = new MemoryCatalogRepository();
|
||||
const database = await catalogRepository.create({
|
||||
workspaceId: "default",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "public",
|
||||
binding: {
|
||||
transport: "postgres_direct",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
});
|
||||
await catalogRepository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [],
|
||||
columns: [],
|
||||
relationships: [],
|
||||
});
|
||||
const started = await catalogRepository.beginPreprocessing("default", "sha256:previous");
|
||||
expect(started.kind).toBe("started");
|
||||
await catalogRepository.finishPreprocessing(
|
||||
"default",
|
||||
0,
|
||||
"sha256:previous",
|
||||
{ status: "succeeded" },
|
||||
);
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: { sessionNew, workspaceInputFingerprint } as any,
|
||||
catalogRepository,
|
||||
workspaceRegistry: {
|
||||
acquireSessionRevision: async () => ({
|
||||
workspace: operationalWorkspace("default"),
|
||||
revision,
|
||||
abort: async () => {},
|
||||
markPersisted: async () => {},
|
||||
}),
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "default", thinking: "low" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { interactionLanguage: "en", question: "q", workspaceId: "default" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toMatchObject({ code: "preprocessing_required" });
|
||||
expect(workspaceInputFingerprint).toHaveBeenCalledWith(revision.snapshotPath);
|
||||
expect(sessionNew).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("rejects an SSH-only Catalog binding before persisting or starting a session", async () => {
|
||||
const sessionNew = vi.fn(async () => ({ id: "must-not-exist" }));
|
||||
const workspaceInputFingerprint = vi.fn(async () => "sha256:unused");
|
||||
const ensure = vi.fn(async () => ({ ok: true }));
|
||||
const createFor = vi.fn();
|
||||
const abort = vi.fn(async () => {});
|
||||
const markPersisted = vi.fn(async () => {});
|
||||
const catalogRepository = new MemoryCatalogRepository();
|
||||
await catalogRepository.create({
|
||||
workspaceId: "ssh-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "postgres",
|
||||
schema: "public",
|
||||
binding: {
|
||||
transport: "ssh_tunnel",
|
||||
username: "reader",
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 22,
|
||||
sshUsername: "tunnel",
|
||||
sshTargetHost: "postgres.internal",
|
||||
sshTargetPort: 5432,
|
||||
},
|
||||
});
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: { sessionNew, searchPack: async () => {} } as any,
|
||||
thtRunner: { sessionNew, searchPack: async () => {}, workspaceInputFingerprint } as any,
|
||||
catalogRepository,
|
||||
readiness: { ensure } as any,
|
||||
mgr: { get: () => undefined, createFor } as any,
|
||||
getSettings: () => ({ workspace: "ssh-workspace" }) as any,
|
||||
workspaceRuntimeSupport: vi.fn(() => false),
|
||||
workspaceRuntimeSupport: vi.fn(() => true),
|
||||
workspaceRegistry: {
|
||||
acquireSessionRevision: vi.fn(async () => ({
|
||||
workspace: {
|
||||
workspace: { schema_version: 4, id: "ssh-workspace", name: "SSH", language: "en" },
|
||||
dwh: {
|
||||
engine: "postgres", database: "postgres", schema: "public",
|
||||
supported_transports: ["ssh_tunnel"],
|
||||
},
|
||||
},
|
||||
workspace: operationalWorkspace("ssh-workspace"),
|
||||
revision: {
|
||||
id: "ssh-workspace", commit: "a".repeat(40), blob: "b".repeat(40),
|
||||
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/ssh-workspace.yaml`,
|
||||
@@ -605,11 +893,12 @@ test("rejects an SSH-only workspace before persisting or starting a session", as
|
||||
} as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toMatchObject({ code: "workspace_not_activatable" });
|
||||
expect(ensure).not.toHaveBeenCalled();
|
||||
expect(workspaceInputFingerprint).not.toHaveBeenCalled();
|
||||
expect(sessionNew).not.toHaveBeenCalled();
|
||||
expect(createFor).not.toHaveBeenCalled();
|
||||
expect(abort).toHaveBeenCalledOnce();
|
||||
@@ -644,7 +933,7 @@ test("hands a revision lease to retention only after the session manifest is dur
|
||||
workspaceRegistry: { acquireSessionRevision } as any,
|
||||
});
|
||||
|
||||
const request = app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const request = app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(markPersisted).not.toHaveBeenCalled();
|
||||
expect(abort).not.toHaveBeenCalled();
|
||||
@@ -675,7 +964,7 @@ test("creates a session from the configured default workspace revision when work
|
||||
workspaceRegistry: registry as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(registry.read).toHaveBeenCalledWith("psd-clinical");
|
||||
expect(sessionNew).toHaveBeenCalledWith(expect.objectContaining({
|
||||
@@ -761,7 +1050,7 @@ test("session lifecycle locates a B session when installation default is A", asy
|
||||
} as any,
|
||||
});
|
||||
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: {
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en",
|
||||
question: "B question", workspaceId: "b-workspace", provider: "zai", model: "glm-5.2", thinking: "low",
|
||||
} })).statusCode).toBe(200);
|
||||
expect((await app.inject({ method: "GET", url: "/sessions" })).json()).toEqual([
|
||||
@@ -773,7 +1062,7 @@ test("session lifecycle locates a B session when installation default is A", asy
|
||||
|
||||
active = undefined;
|
||||
expect((await app.inject({ method: "POST", url: "/sessions/session-b/resume" })).json())
|
||||
.toEqual({ id: "session-b", alreadyActive: false });
|
||||
.toEqual({ id: "session-b", alreadyActive: false, workspaceId: "b-workspace" });
|
||||
expect(calls).toContain(`new:${bPath}`);
|
||||
expect(calls).toContain(`list:${bPath}`);
|
||||
expect(calls).toContain(`show:${bPath}`);
|
||||
@@ -804,7 +1093,7 @@ test("POST /sessions uses the catalog default with workspace/thinking settings a
|
||||
],
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const created = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const created = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(created.json()).toEqual({ id: "s1" });
|
||||
expect(sessionNewArg.workspaceConfigPath).toContain(`/snapshots/${"e".repeat(40)}/w.yaml`);
|
||||
expect(sessionNewArg.provider).toBe("zai");
|
||||
@@ -865,11 +1154,11 @@ test("POST /sessions stops the user's previous open Pi runtime before creating a
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { question: "one" } })).statusCode)
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "one" } })).statusCode)
|
||||
.toBe(200);
|
||||
order.length = 0;
|
||||
|
||||
const second = await app.inject({ method: "POST", url: "/sessions", payload: { question: "two" } });
|
||||
const second = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "two" } });
|
||||
|
||||
expect(second.statusCode).toBe(200);
|
||||
expect(second.json()).toEqual({ id: "s2" });
|
||||
@@ -890,7 +1179,7 @@ test("POST /sessions refuses to create a session when the local DWH precheck fai
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toMatchObject({ code: "dwh_unreachable" });
|
||||
expect(pinged).toBe(1);
|
||||
@@ -911,7 +1200,7 @@ test("POST /sessions proceeds past a passing DWH precheck", async () => {
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(pinged).toBe(1);
|
||||
expect(created).toBe(1);
|
||||
@@ -930,7 +1219,7 @@ test("POST /sessions skips the DWH precheck when the flag is off (default)", asy
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(pinged).toBe(0); // probe never runs without the flag
|
||||
});
|
||||
@@ -964,7 +1253,7 @@ test("POST /sessions configura Pi con il thinking globale selezionato", async ()
|
||||
],
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(configured.thinking).toBe("high");
|
||||
@@ -1043,6 +1332,29 @@ test("POST /sessions/:id/resume uses the manifest's retained workspace revision"
|
||||
);
|
||||
});
|
||||
|
||||
test.each([undefined, { provider: "local", model: "qwen" }])("resume uses the global model/default, not the historical manifest (%j)", async (payload) => {
|
||||
const configure = vi.fn(async () => {});
|
||||
const manifest = { status: "open", archived: false, provider: "retired", model: "old-model" };
|
||||
const runtime = { bridge: { onClientEvent: () => {}, emitClientEvent: () => {} } };
|
||||
let current: any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => current,
|
||||
createFor: () => { current = runtime; return runtime; },
|
||||
configure, start: () => {},
|
||||
} as any,
|
||||
thtRunner: { sessionShow: async () => manifest, reopenSession: async () => {} } as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
runtimeModelCatalog: sessionCatalog("zai/glm-5.2", ["zai/glm-5.2", "local/qwen"]),
|
||||
getSettings: () => ({ workspace: "psd", thinking: "medium" }) as any,
|
||||
});
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/model-resume/resume", ...(payload ? { payload } : {}) });
|
||||
expect(response.statusCode).toBe(200);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(configure).toHaveBeenCalledWith(expect.anything(), expect.objectContaining(payload ?? { provider: "zai", model: "glm-5.2" }));
|
||||
expect(manifest).toMatchObject({ provider: "retired", model: "old-model" });
|
||||
});
|
||||
|
||||
test("POST /sessions/:id/resume returns a sanitized error when its retained revision is unavailable", async () => {
|
||||
const rawFailure = "cannot read /data/workspace-registry/snapshots/secret-revision";
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
@@ -1269,7 +1581,7 @@ test("resuming a different session stops the user's previous Pi runtime", async
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { question: "one" } })).statusCode)
|
||||
expect((await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "one" } })).statusCode)
|
||||
.toBe(200);
|
||||
|
||||
const resumed = await app.inject({ method: "POST", url: "/sessions/s2/resume" });
|
||||
@@ -1657,13 +1969,13 @@ test.each([
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "old" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "old" } });
|
||||
if (lifecycle === "close") {
|
||||
await app.inject({ method: "POST", url: "/sessions/s1/close" });
|
||||
await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
} else {
|
||||
await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "replacement" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "replacement" } });
|
||||
}
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(current).toBe(replacement);
|
||||
@@ -1727,7 +2039,7 @@ test.each(["resolve", "reject"] as const)(
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "old" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "old" } });
|
||||
await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
published.length = 0;
|
||||
|
||||
@@ -1919,7 +2231,7 @@ test("Close suppresses a bootstrap that settles while close persistence is pendi
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
const closeResponse = app.inject({ method: "POST", url: "/sessions/s1/close" });
|
||||
await closeStarted;
|
||||
published.length = 0;
|
||||
@@ -1993,7 +2305,7 @@ test("bootstrap failure persists once and keeps Resume serialized behind that pe
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
oldConfigure.reject(new Error("configure failed"));
|
||||
await failureStarted;
|
||||
const resumeResponse = app.inject({ method: "POST", url: "/sessions/s1/resume" })
|
||||
@@ -2116,7 +2428,7 @@ test("a replaced runtime cannot publish or fail the newly resumed session", asyn
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
oldBridge.setState("idle");
|
||||
|
||||
@@ -2176,7 +2488,7 @@ test("a deleted runtime cannot repopulate or fail the forgotten session", async
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
const response = await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
@@ -2221,7 +2533,7 @@ test("an unexpectedly exited runtime publishes its terminal sequence then releas
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
published.length = 0;
|
||||
|
||||
@@ -2272,7 +2584,7 @@ test("agent_end releases the Pi runtime after the session was finalized", async
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
|
||||
@@ -2344,7 +2656,7 @@ test("POST /sessions/:id/response senza gate pendente risponde 409 (risposta sta
|
||||
getSettings: () => ({ workspace: "w" }),
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
// The fake Pi never emitted a ui_request: the bridge has no pending descriptor, so a
|
||||
// response (stale UI, double submit) must be rejected instead of forwarded to Pi.
|
||||
const res = await app.inject({ method: "POST", url: "/sessions/s1/response",
|
||||
@@ -2485,7 +2797,7 @@ test("POST /sessions readiness failure returns one fixed public message without
|
||||
getSettings: () => ({ workspace: "psd" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toEqual({
|
||||
error: "Session services are not ready. Check configuration and connectivity, then try again.",
|
||||
@@ -2506,7 +2818,7 @@ test.each(["semantic_index_incompatible", "workspace_not_activatable"] as const)
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST", url: "/sessions", payload: { question: "q" },
|
||||
method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
@@ -2529,7 +2841,7 @@ test("POST /sessions returns storage 503 before creating a Pi runtime when sessi
|
||||
mgr: { createFor: () => { piCreated = true; throw new Error("must not spawn"); } } as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.json()).toEqual({ error: "session storage is unavailable" });
|
||||
@@ -2549,13 +2861,13 @@ test("POST /sessions proceeds when ollamaEnsure succeeds", async () => {
|
||||
getSettings: () => ({ workspace: "psd" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.json()).toEqual({ id: "s1" });
|
||||
expect(qdrantEnsure).toHaveBeenCalledWith(operationalWorkspace("psd"), 60, "self_heal");
|
||||
expect(ensureWs).toContain(`/snapshots/${"e".repeat(40)}/psd.yaml`);
|
||||
});
|
||||
|
||||
test("POST /sessions falls back from a stale requested model to the catalog default", async () => {
|
||||
test("POST /sessions rejects a stale requested model without silently using the default", async () => {
|
||||
let created = 0;
|
||||
let persisted: any;
|
||||
const runtime = { bridge: { onClientEvent: () => {} } };
|
||||
@@ -2582,16 +2894,13 @@ test("POST /sessions falls back from a stale requested model to the catalog defa
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "q", provider: "deepseek", model: "deepseek-v4-pro" },
|
||||
payload: { interactionLanguage: "en", question: "q", provider: "deepseek", model: "deepseek-v4-pro" },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json()).toEqual({
|
||||
id: "fallback",
|
||||
warning: "Configured model deepseek/deepseek-v4-pro is unavailable; using zai/glm-5.2.",
|
||||
});
|
||||
expect(persisted).toMatchObject({ provider: "zai", model: "glm-5.2" });
|
||||
expect(created).toBe(1);
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toMatchObject({ code: "model_unavailable" });
|
||||
expect(persisted).toBeUndefined();
|
||||
expect(created).toBe(0);
|
||||
});
|
||||
|
||||
test("POST /sessions marks a persisted session failed when runtime construction throws", async () => {
|
||||
@@ -2620,7 +2929,7 @@ test("POST /sessions marks a persisted session failed when runtime construction
|
||||
],
|
||||
});
|
||||
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(res.statusCode).toBe(503);
|
||||
expect(res.json()).toEqual({
|
||||
@@ -2713,7 +3022,7 @@ test.each([
|
||||
|
||||
try {
|
||||
const response = flow === "new"
|
||||
? await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } })
|
||||
? await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } })
|
||||
: await app.inject({ method: "POST", url: `/sessions/${sessionId}/resume` });
|
||||
const logs = consoleError.mock.calls.flat().map(String).join(" ");
|
||||
|
||||
@@ -2819,7 +3128,7 @@ test("POST /sessions returns after bridge attachment but starts only after retri
|
||||
getSettings: () => ({ workspace: "psd" }) as any,
|
||||
});
|
||||
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const res = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
expect(res.json()).toEqual({ id: "s-early" });
|
||||
expect(bridgeAttached).toBe(true);
|
||||
expect(started).toBe(false);
|
||||
@@ -2866,7 +3175,7 @@ test("POST /sessions bootstrap failure emits only a fixed recovery message", asy
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "q" },
|
||||
payload: { interactionLanguage: "en", question: "q" },
|
||||
});
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
@@ -2962,7 +3271,7 @@ test.each([
|
||||
})),
|
||||
} as any,
|
||||
});
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
const response = await app.inject({ method: "POST", url: "/sessions", payload: { interactionLanguage: "en", question: "q" } });
|
||||
|
||||
expect(response.statusCode).toBe(expectedStatus);
|
||||
expect(ensure).toHaveBeenCalledTimes(reachesReadiness ? 1 : 0);
|
||||
|
||||
@@ -160,8 +160,7 @@ test("GET /models returns session choices from the installation model catalog",
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {} as any,
|
||||
runtimeModelCatalog: {
|
||||
defaultSession: "zai/glm-5.2",
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: "zai/glm-5.2",
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
sessionModels: () => [{
|
||||
id: "zai/glm-5.2", provider: "zai", model: "glm-5.2", label: "GLM 5.2",
|
||||
@@ -187,8 +186,7 @@ test("GET /models returns an empty list when the catalog has no session models",
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {} as any,
|
||||
runtimeModelCatalog: {
|
||||
defaultSession: null,
|
||||
defaultMetadataGeneration: null,
|
||||
defaultInteraction: null,
|
||||
embedding: null,
|
||||
sessionModels: () => [],
|
||||
metadataModels: () => [],
|
||||
|
||||
@@ -500,8 +500,9 @@ async function createRealRouteFixture() {
|
||||
await git(author, ["init", "--initial-branch=main"]);
|
||||
await git(author, ["config", "user.name", "Workspace Route Test"]);
|
||||
await git(author, ["config", "user.email", "workspace-route@example.invalid"]);
|
||||
const { dwh: _runtimeDwh, diagnostics: _runtimeDiagnostics, ...authoredWorkspace } = workspace;
|
||||
const descriptor: CanonicalWorkspace = {
|
||||
...workspace,
|
||||
...authoredWorkspace,
|
||||
evidence: {
|
||||
source: {
|
||||
type: "filesystem",
|
||||
|
||||
@@ -52,8 +52,11 @@ test("Qdrant readiness uses only the internal URL and accepts the exact collecti
|
||||
const request = vi.fn(async () => response(200, collection()));
|
||||
|
||||
await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ ok: true, state: "ready" });
|
||||
expect(request).toHaveBeenCalledOnce();
|
||||
expect(request.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd");
|
||||
expect(request).toHaveBeenCalledTimes(2);
|
||||
expect(request.mock.calls.map((call) => call[0])).toEqual([
|
||||
"http://qdrant:6333/collections/psd-reference",
|
||||
"http://qdrant:6333/collections/psd-memory",
|
||||
]);
|
||||
expect(request.mock.calls[0][1]).toMatchObject({ method: "GET", signal: expect.any(AbortSignal) });
|
||||
});
|
||||
|
||||
|
||||
@@ -34,6 +34,19 @@ test("sessionNew parses id from JSON", async () => {
|
||||
expect(await r.sessionNew({ question: "q" })).toEqual({ id: "2026-06-27-100000-x" });
|
||||
});
|
||||
|
||||
test("session language uses public per-command CLI flags and the selected config", async () => {
|
||||
const runner = new ThtRunner({ thtBin: "tht", harnessDir: "/h", configPath: "config/tht.yaml" });
|
||||
(spawn as any).mockClear();
|
||||
await runner.sessionNew({ question: "Pazienti", interactionLanguage: "en" });
|
||||
expect((spawn as any).mock.calls[0][1]).toEqual([
|
||||
"session", "new", "Pazienti", "--interaction-language", "en", "--json", "-c", "config/tht.yaml",
|
||||
]);
|
||||
await runner.ensureInteractionLanguage("s1");
|
||||
expect((spawn as any).mock.calls[1][1]).toEqual([
|
||||
"session", "ensure-interaction-language", "s1", "--json", "-c", "config/tht.yaml",
|
||||
]);
|
||||
});
|
||||
|
||||
test("searchPack persists retrieval context with session and workspace", async () => {
|
||||
const calls: any[] = [];
|
||||
const r = new ThtRunner({ thtBin: "tht", harnessDir: "/h", configPath: "config/tht.yaml" });
|
||||
|
||||
@@ -30,11 +30,11 @@ function ok(operation: string): WorkspaceOperationResult {
|
||||
};
|
||||
}
|
||||
|
||||
test("entrypoint emits exactly one pristine JSON document and maps success/block/failure exits", async () => {
|
||||
test("entrypoint emits exactly one pristine JSON document and maps success/failure exits", async () => {
|
||||
const service = {
|
||||
inspect: vi.fn(async () => ok("inspect")),
|
||||
preprocessDwh: vi.fn(async () => ({ ...ok("preprocess dwh"), status: "blocked", code: "manual_review_required" as const })),
|
||||
run: vi.fn(async () => ({ ...ok("preprocess run"), status: "failed", code: "semantic_index_incompatible" as const })),
|
||||
clear: vi.fn(async () => ok("preprocess clear")),
|
||||
} as any;
|
||||
|
||||
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
@@ -42,13 +42,13 @@ test("entrypoint emits exactly one pristine JSON document and maps success/block
|
||||
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "inspect", code: "ok" });
|
||||
expect(inspectIo.stderr.join("")).toBe("");
|
||||
|
||||
const blockedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-dwh"], service, blockedIo)).toBe(3);
|
||||
expect(JSON.parse(blockedIo.stdout.join(""))).toMatchObject({ code: "manual_review_required" });
|
||||
|
||||
const failedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-run"], service, failedIo)).toBe(1);
|
||||
expect(JSON.parse(failedIo.stdout.join(""))).toMatchObject({ code: "semantic_index_incompatible" });
|
||||
|
||||
const clearIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-clear"], service, clearIo)).toBe(0);
|
||||
expect(JSON.parse(clearIo.stdout.join(""))).toMatchObject({ operation: "preprocess clear", code: "ok" });
|
||||
});
|
||||
|
||||
test("malformed stdin, unknown commands, and extra fields fail with exit 2 but still return bounded JSON", async () => {
|
||||
@@ -84,31 +84,19 @@ test("raw exception text is redacted from stderr and stdout remains within the p
|
||||
expect(captured.stderr.join("")).not.toContain("SELECT *");
|
||||
});
|
||||
|
||||
test("vector-inspect and vector-rebuild dispatch to the service with the exact envelope", async () => {
|
||||
test("retired partial preprocessing commands are rejected without dispatch", async () => {
|
||||
const service = {
|
||||
vectorInspect: vi.fn(async () => ok("vector inspect")),
|
||||
vectorRebuild: vi.fn(async () => ok("vector rebuild")),
|
||||
preprocessDwh: vi.fn(),
|
||||
vectorInspect: vi.fn(),
|
||||
vectorRebuild: vi.fn(),
|
||||
} as any;
|
||||
|
||||
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-inspect"], service, inspectIo)).toBe(0);
|
||||
expect(service.vectorInspect).toHaveBeenCalledWith({ workspaceId: "psd-clinical" });
|
||||
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "vector inspect", code: "ok" });
|
||||
|
||||
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(0);
|
||||
expect(service.vectorRebuild).toHaveBeenCalledWith({
|
||||
workspaceId: "psd-clinical",
|
||||
collection: "psd-clinical",
|
||||
confirm: "psd-clinical",
|
||||
destroy: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("vector-rebuild without exact confirmation is refused by the service", async () => {
|
||||
const service = {
|
||||
vectorRebuild: vi.fn(async () => ({ ...ok("vector rebuild"), status: "failed", code: "semantic_index_incompatible" as const })),
|
||||
} as any;
|
||||
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "other", confirm: "other", destroy: true }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(1);
|
||||
for (const command of ["preprocess-dwh", "vector-inspect", "vector-rebuild"]) {
|
||||
const captured = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", command], service, captured)).toBe(2);
|
||||
expect(JSON.parse(captured.stdout.join(""))).toMatchObject({ status: "failed", operation: command });
|
||||
}
|
||||
expect(service.preprocessDwh).not.toHaveBeenCalled();
|
||||
expect(service.vectorInspect).not.toHaveBeenCalled();
|
||||
expect(service.vectorRebuild).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
import Fastify from "fastify";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
import type { CatalogRepository, WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import {
|
||||
readWorkspacePreprocessingStatus,
|
||||
workspacePreprocessingRoutes,
|
||||
type WorkspacePreprocessingRouteDeps,
|
||||
} from "../src/routes/workspace-preprocessing.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
|
||||
const admin: PrincipalContext = {
|
||||
issuer: "test",
|
||||
subject: "admin",
|
||||
roles: ["admin"],
|
||||
permissions: ["session.use", "database.manage"],
|
||||
isAdmin: true,
|
||||
};
|
||||
|
||||
function database(overrides: Partial<WorkspaceDatabase> = {}): WorkspaceDatabase {
|
||||
return {
|
||||
id: "49b6491a-78bb-4cee-b27b-0efaf4419774",
|
||||
workspaceId: "catalog-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "analytics",
|
||||
version: 4,
|
||||
createdAt: "2026-09-05T10:00:00.000Z",
|
||||
updatedAt: "2026-09-05T10:00:00.000Z",
|
||||
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
|
||||
connectionStatus: "reachable",
|
||||
schemaSyncedVersion: 4,
|
||||
metadataContentRevision: 18,
|
||||
preprocessingStatus: "failed",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function routeDeps(
|
||||
current: () => WorkspaceDatabase | undefined,
|
||||
overrides: Partial<WorkspacePreprocessingRouteDeps> = {},
|
||||
): WorkspacePreprocessingRouteDeps {
|
||||
const repository = {
|
||||
getByWorkspace: vi.fn(async () => current()),
|
||||
listSyncRuns: vi.fn(async () => []),
|
||||
getActiveDescriptionGenerationRun: vi.fn(async () => undefined),
|
||||
listSensitivityAnalysisRuns: vi.fn(async () => []),
|
||||
} as unknown as CatalogRepository;
|
||||
const registry = {
|
||||
read: vi.fn(async () => ({
|
||||
workspace: {
|
||||
workspace: {
|
||||
schema_version: 4,
|
||||
id: "catalog-workspace",
|
||||
name: "Catalog workspace",
|
||||
language: "en",
|
||||
},
|
||||
},
|
||||
revision: {
|
||||
id: "catalog-workspace",
|
||||
commit: "a".repeat(40),
|
||||
blob: "b".repeat(40),
|
||||
snapshotPath: "/data/workspaces/catalog-workspace.yaml",
|
||||
},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
return {
|
||||
repository,
|
||||
registry,
|
||||
service: {
|
||||
run: vi.fn(async () => ({ status: "succeeded" })),
|
||||
clear: vi.fn(async () => ({ status: "succeeded" })),
|
||||
} as never,
|
||||
inputFingerprint: {
|
||||
workspaceInputFingerprint: vi.fn(async () => "sha256:current"),
|
||||
} as never,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("reports ready only when the Catalog revision and runtime fingerprint are current", async () => {
|
||||
const ready = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
|
||||
});
|
||||
await expect(readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => ready),
|
||||
)).resolves.toMatchObject({
|
||||
state: "ready",
|
||||
actionable: true,
|
||||
clearable: true,
|
||||
detail: "Catalog revision 18 is indexed.",
|
||||
});
|
||||
|
||||
ready.preprocessingInputFingerprint = "sha256:old";
|
||||
await expect(readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => ready),
|
||||
)).resolves.toMatchObject({
|
||||
state: "required",
|
||||
actionable: true,
|
||||
clearable: true,
|
||||
detail: "Catalog revision 18 is not indexed.",
|
||||
});
|
||||
});
|
||||
|
||||
test("explains a current blocked prerequisite without inventing a run log", async () => {
|
||||
const status = await readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => database({ schemaSyncedVersion: 3 })),
|
||||
);
|
||||
expect(status).toMatchObject({
|
||||
state: "blocked",
|
||||
actionable: false,
|
||||
clearable: true,
|
||||
detail: "Catalog synchronization is required.",
|
||||
reason: "Database configuration v4 is newer than the latest Catalog synchronization v3.",
|
||||
nextStep: "Open Database management and run Synchronize schema.",
|
||||
});
|
||||
expect(status).not.toHaveProperty("lastFailure");
|
||||
});
|
||||
|
||||
test("exposes only the latest sanitized failed-run diagnostic", async () => {
|
||||
const status = await readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => database({
|
||||
preprocessingStatus: "failed",
|
||||
preprocessingErrorCode: "schema_index_failed",
|
||||
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
|
||||
})),
|
||||
);
|
||||
expect(status).toMatchObject({
|
||||
state: "failed",
|
||||
actionable: true,
|
||||
clearable: true,
|
||||
reason: expect.stringContaining("schema indexing worker"),
|
||||
lastFailure: {
|
||||
stage: "schema_index",
|
||||
errorCode: "schema_index_failed",
|
||||
finishedAt: "2026-09-05T10:04:00.000Z",
|
||||
},
|
||||
});
|
||||
expect(status).not.toHaveProperty("history");
|
||||
});
|
||||
|
||||
test("reports the durable phase of the active preprocessing run", async () => {
|
||||
const status = await readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(
|
||||
() => database({ preprocessingStatus: "running" }),
|
||||
{
|
||||
readLatestJob: () => ({
|
||||
status: "active",
|
||||
completedStages: ["catalog_snapshot"],
|
||||
}) as never,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(status).toMatchObject({
|
||||
state: "running",
|
||||
detail: "Building schema vectors and LSH indexes.",
|
||||
progress: { stage: "schema_index", step: 2, totalSteps: 4 },
|
||||
});
|
||||
});
|
||||
|
||||
test("explicitly reruns preprocessing when the current Catalog input is already ready", async () => {
|
||||
const ready = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
});
|
||||
const deps = routeDeps(() => ready);
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async (request) => { request.principal = admin; });
|
||||
workspacePreprocessingRoutes(app, deps);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/workspaces/catalog-workspace/preprocessing",
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
|
||||
expect(deps.service.run).toHaveBeenCalledTimes(1);
|
||||
await app.close();
|
||||
});
|
||||
|
||||
test("runs preprocessing once from the sanctioned admin endpoint and returns the new state", async () => {
|
||||
let current = database();
|
||||
const deps = routeDeps(() => current, {
|
||||
service: {
|
||||
run: vi.fn(async () => {
|
||||
current = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
|
||||
});
|
||||
return { status: "succeeded" } as never;
|
||||
}),
|
||||
clear: vi.fn(async () => ({ status: "succeeded" } as never)),
|
||||
},
|
||||
});
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async (request) => { request.principal = admin; });
|
||||
workspacePreprocessingRoutes(app, deps);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/workspaces/catalog-workspace/preprocessing",
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
|
||||
expect(deps.service.run).toHaveBeenCalledTimes(1);
|
||||
await app.close();
|
||||
});
|
||||
|
||||
test("clears only derived preprocessing data from the sanctioned admin endpoint", async () => {
|
||||
let current = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
});
|
||||
const deps = routeDeps(() => current, {
|
||||
service: {
|
||||
run: vi.fn(),
|
||||
clear: vi.fn(async () => {
|
||||
current = database({
|
||||
preprocessingStatus: "failed",
|
||||
preprocessingErrorCode: "derived_data_cleared",
|
||||
preprocessingFinishedAt: "2026-09-05T10:05:00.000Z",
|
||||
});
|
||||
return { status: "succeeded" } as never;
|
||||
}),
|
||||
},
|
||||
});
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async (request) => { request.principal = admin; });
|
||||
workspacePreprocessingRoutes(app, deps);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "DELETE",
|
||||
url: "/workspaces/catalog-workspace/preprocessing",
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({
|
||||
state: "required",
|
||||
clearable: false,
|
||||
detail: "Reference vectors and LSH are empty. Memory is preserved.",
|
||||
});
|
||||
expect(deps.service.clear).toHaveBeenCalledTimes(1);
|
||||
await app.close();
|
||||
});
|
||||
@@ -2,9 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { PreprocessingStateStore } from "../src/workspaces/preprocessing-state.js";
|
||||
import { syncAnnotations } from "../src/workspaces/annotations-sync.js";
|
||||
import { validateWorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
import {
|
||||
WorkspacePreprocessingService,
|
||||
type ChildProcessRequest,
|
||||
@@ -12,77 +10,51 @@ import {
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
test("Evidence consolidation runs only its stage and preserves blocked Catalog readiness", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", counts: { documents: 2 } }), stderr: "" }));
|
||||
const result = await service({ repository: catalog, runChild }).consolidateEvidence({ workspaceId: "catalog-workspace" });
|
||||
expect(result.status).toBe("succeeded");
|
||||
expect(runChild).toHaveBeenCalledOnce();
|
||||
expect(runChild.mock.calls[0]?.[0]).toMatchObject({ argv: ["preprocess", "evidence", "--consolidate", "--json", "-c", "/dev/fd/3"] });
|
||||
expect(catalog.beginPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.clearPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("Evidence index failure reports retry without changing Catalog state", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({ exitCode: 1, stdout: JSON.stringify({ status: "failed", saved: true, error: "Saved; retry indexing." }), stderr: "private provider endpoint" }));
|
||||
const result = await service({ repository: catalog, runChild }).consolidateEvidence({ workspaceId: "catalog-workspace" });
|
||||
expect(result.status).toBe("failed");
|
||||
expect(result.warnings).toEqual(["Saved; retry indexing."]);
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(result)).not.toContain("private provider");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
|
||||
});
|
||||
|
||||
const semanticRuntime = {
|
||||
internalQdrantUrl: "http://qdrant:6333",
|
||||
internalEmbeddingUrl: "http://embedding:11434",
|
||||
internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
|
||||
internalEmbeddingModel: "qwen3-embedding:0.6b",
|
||||
internalEmbeddingDimensions: 1024,
|
||||
};
|
||||
const workspace = validateWorkspaceDescriptor({
|
||||
workspace: {
|
||||
schema_version: 4,
|
||||
id: "catalog-workspace",
|
||||
name: "Catalog only",
|
||||
language: "en",
|
||||
},
|
||||
});
|
||||
|
||||
const baseWorkspace = parseWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Runtime Lease
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
`);
|
||||
const filesystemWorkspace = parseWorkspaceYaml(`${baseWorkspace ? '' : ''}workspace:
|
||||
schema_version: 4
|
||||
id: fs-workspace
|
||||
name: Filesystem
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: filesystem
|
||||
uri: fs-workspace/evidence
|
||||
`);
|
||||
const privateHttpWorkspace = parseWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: http-workspace
|
||||
name: Http
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: http
|
||||
uris: [http://127.0.0.1/private.md]
|
||||
authentication: none
|
||||
connect_timeout_ms: 1000
|
||||
read_timeout_ms: 2000
|
||||
max_bytes: 100
|
||||
max_redirects: 0
|
||||
allow_private_hosts: true
|
||||
max_cache_bytes: 100
|
||||
`);
|
||||
|
||||
function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id) {
|
||||
function runtime() {
|
||||
return {
|
||||
workspace,
|
||||
workspaceId,
|
||||
workspaceId: "catalog-workspace",
|
||||
workspaceRevision: "a".repeat(40),
|
||||
descriptorBlob: "b".repeat(40),
|
||||
catalogBlob: "c".repeat(40),
|
||||
configLease: {
|
||||
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}-identitysuffix.yaml`,
|
||||
workspaceId,
|
||||
path: `/data/sessions/catalog-workspace/preprocessing/runtime-config/${"a".repeat(40)}.yaml`,
|
||||
workspaceId: "catalog-workspace",
|
||||
workspaceRevision: "a".repeat(40),
|
||||
descriptorBlob: "b".repeat(40),
|
||||
catalogBlob: "c".repeat(40),
|
||||
@@ -90,21 +62,32 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
|
||||
bindingDigest: "sha256:bindings",
|
||||
semanticQdrantUrl: "http://qdrant:6333",
|
||||
effectiveConfig: {
|
||||
schemaVersion: 2,
|
||||
schemaVersion: 3,
|
||||
dwh: {
|
||||
engine: "postgres",
|
||||
database: "analytics",
|
||||
schema: "mart",
|
||||
database: "warehouse",
|
||||
schema: "analytics",
|
||||
transport: "postgres_direct",
|
||||
host: "dwh.internal",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
user: "reader",
|
||||
},
|
||||
vector: { collection: workspaceId, dimensions: 1024, distance: "cosine" },
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
vector: {
|
||||
collections: {
|
||||
reference: "catalog-workspace-reference",
|
||||
memory: "catalog-workspace-memory",
|
||||
},
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
},
|
||||
embedding: {
|
||||
id: "ollama/qwen3-embedding:0.6b",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
roots: { artifacts: "/data/artifacts", indexes: "/data/indexes" },
|
||||
},
|
||||
effectiveConfigIdentity: "workspace://psd-clinical@v1:" + "d".repeat(64),
|
||||
effectiveConfigIdentity: "workspace://catalog-workspace@v1:" + "d".repeat(64),
|
||||
configFingerprint: "sha256:" + "e".repeat(64),
|
||||
inputFingerprint: "sha256:" + "f".repeat(64),
|
||||
release: () => undefined,
|
||||
@@ -112,385 +95,226 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(workspace = baseWorkspace) {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
|
||||
roots.push(dataRoot);
|
||||
const requests: ChildProcessRequest[] = [];
|
||||
const runChild = vi.fn(async (request: ChildProcessRequest) => {
|
||||
requests.push(request);
|
||||
return { exitCode: 0, stdout: JSON.stringify({ status: "succeeded" }), stderr: "" };
|
||||
});
|
||||
const service = new WorkspacePreprocessingService({
|
||||
dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(workspace),
|
||||
runChild,
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: true }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
});
|
||||
return { dataRoot, runChild, requests, service };
|
||||
const database = {
|
||||
id: "database-1",
|
||||
workspaceId: "catalog-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "analytics",
|
||||
version: 4,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
|
||||
connectionStatus: "reachable",
|
||||
schemaSyncedVersion: 4,
|
||||
metadataContentRevision: 9,
|
||||
preprocessingStatus: "running",
|
||||
} as const;
|
||||
|
||||
function repository(overrides: Record<string, unknown> = {}) {
|
||||
const finishPreprocessing = vi.fn(async () => ({
|
||||
...database,
|
||||
preprocessingStatus: "succeeded" as const,
|
||||
preprocessedMetadataRevision: 9,
|
||||
}));
|
||||
return {
|
||||
beginPreprocessing: vi.fn(async () => ({ kind: "started" as const, database })),
|
||||
finishPreprocessing,
|
||||
clearPreprocessing: vi.fn(async () => ({ kind: "cleared" as const, database })),
|
||||
getByWorkspace: vi.fn(async () => database),
|
||||
listTables: vi.fn(async () => [{
|
||||
id: "table-1", databaseId: database.id, name: "orders",
|
||||
description: "Curated orders", generatedDescription: "Generated orders",
|
||||
sourceComment: "PostgreSQL orders", version: 1,
|
||||
createdAt: database.createdAt, updatedAt: database.updatedAt,
|
||||
lastSyncedDatabaseVersion: 4, lastSyncedAt: database.updatedAt,
|
||||
}]),
|
||||
listColumns: vi.fn(async () => [{
|
||||
id: "column-1", tableId: "table-1", name: "customer_id", ordinalPosition: 1,
|
||||
dataType: "uuid", isNullable: false, defaultExpression: null,
|
||||
primaryKeyPosition: null, isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1,
|
||||
sourceComment: "PostgreSQL customer", description: null,
|
||||
generatedDescription: "Generated customer", sensitive: true,
|
||||
sensitivityReason: "identifier", lastSyncedDatabaseVersion: 4,
|
||||
lastSyncedAt: database.updatedAt, version: 1,
|
||||
createdAt: database.createdAt, updatedAt: database.updatedAt,
|
||||
}]),
|
||||
listRelationships: vi.fn(async () => []),
|
||||
listLogicalRelationships: vi.fn(async () => []),
|
||||
...overrides,
|
||||
} as any;
|
||||
}
|
||||
|
||||
test("preprocess dwh uses fixed argv and resumes outer state without rerunning a completed stage", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const first = await f.service.preprocessDwh({ workspaceId: "psd-clinical" });
|
||||
expect(first).toMatchObject({
|
||||
status: "succeeded",
|
||||
code: "ok",
|
||||
operation: "preprocess dwh",
|
||||
completedStages: ["dwh"],
|
||||
childRuns: { dwh: "d".repeat(32) },
|
||||
});
|
||||
expect((f.runChild.mock.calls[0]![0] as ChildProcessRequest).argv).toEqual([
|
||||
"preprocess", "dwh", "--steps", "introspect,lsh", "--json", "-c", "/dev/fd/3",
|
||||
]);
|
||||
|
||||
const second = await f.service.preprocessDwh({
|
||||
workspaceId: "psd-clinical",
|
||||
resumeRunId: first.runId!,
|
||||
});
|
||||
expect(second.status).toBe("unchanged");
|
||||
expect(f.runChild).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("schema suggest-fks publishes a candidate artifact and blocks full runs for manual review", async () => {
|
||||
const f = fixture();
|
||||
f.runChild
|
||||
.mockResolvedValueOnce({
|
||||
function service(options: {
|
||||
repository?: any;
|
||||
runChild?: (request: ChildProcessRequest) => Promise<{
|
||||
exitCode: number; stdout: string; stderr: string;
|
||||
}>;
|
||||
semanticPreflight?: () => Promise<
|
||||
{ ok: true } | { ok: false; code: "semantic_index_incompatible" }
|
||||
>;
|
||||
} = {}) {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), "tht-catalog-preprocessing-"));
|
||||
roots.push(dataRoot);
|
||||
return new WorkspacePreprocessingService({
|
||||
dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(),
|
||||
runChild: options.runChild ?? (async () => ({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
|
||||
stderr: "",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: []\n",
|
||||
}),
|
||||
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
|
||||
stderr: "",
|
||||
})),
|
||||
semanticPreflight: options.semanticPreflight ?? (async () => ({ ok: true })),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
catalogRepository: options.repository,
|
||||
});
|
||||
}
|
||||
|
||||
test("complete preprocessing snapshots Catalog metadata and commits its PostgreSQL state", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async (request: ChildProcessRequest) => {
|
||||
const snapshotPath = request.argv[request.argv.indexOf("--catalog-metadata") + 1]!;
|
||||
const snapshot = JSON.parse(readFileSync(snapshotPath, "utf8"));
|
||||
expect(snapshot).toMatchObject({
|
||||
workspaceId: "catalog-workspace",
|
||||
databaseName: "warehouse",
|
||||
metadataContentRevision: 9,
|
||||
tables: [{
|
||||
name: "orders",
|
||||
description: "Curated orders",
|
||||
descriptionSource: "curated",
|
||||
columns: [{
|
||||
name: "customer_id",
|
||||
description: "Generated customer",
|
||||
descriptionSource: "generated",
|
||||
sensitive: true,
|
||||
}],
|
||||
}],
|
||||
});
|
||||
|
||||
const result = await f.service.run({ workspaceId: "psd-clinical" });
|
||||
expect(result).toMatchObject({
|
||||
status: "blocked",
|
||||
code: "manual_review_required",
|
||||
completedStages: ["dwh", "fk_suggest"],
|
||||
});
|
||||
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv[0])).toEqual(["preprocess", "schema"]);
|
||||
});
|
||||
|
||||
test("schema check requires the exact candidate digest and stages annotations via a temp file without recording a review", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: []\n",
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
|
||||
await expect(f.service.checkSchema({
|
||||
workspaceId: "psd-clinical",
|
||||
annotationsYaml: "tables: {}\n",
|
||||
reviewedCandidatesDigest: "sha256:" + "f".repeat(64),
|
||||
})).resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
|
||||
const reviewedDigest = suggest.artifactIdentities![0]!.digest;
|
||||
let stagedPath = "";
|
||||
f.runChild.mockImplementationOnce(async (request: ChildProcessRequest) => {
|
||||
stagedPath = request.argv[request.argv.indexOf("--annotations") + 1]!;
|
||||
expect(readFileSync(stagedPath, "utf8")).toBe("tables: {}\n");
|
||||
expect(request.argv).toEqual([
|
||||
"schema", "check", "--annotations", stagedPath,
|
||||
"--reviewed-candidates", reviewedDigest,
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
]);
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
orphan_count: 0,
|
||||
annotations_digest: "sha256:annotations",
|
||||
reviewed_candidates_digest: reviewedDigest,
|
||||
}),
|
||||
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
|
||||
stderr: "",
|
||||
};
|
||||
});
|
||||
|
||||
const checked = await f.service.checkSchema({
|
||||
workspaceId: "psd-clinical",
|
||||
annotationsYaml: "tables: {}\n",
|
||||
reviewedCandidatesDigest: reviewedDigest,
|
||||
});
|
||||
expect(checked).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
expect(() => readFileSync(stagedPath, "utf8")).toThrow();
|
||||
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
|
||||
expect(state.readFkReview(suggest.runId!)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("index schema fails closed when semantic preflight refuses the collection", async () => {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
|
||||
roots.push(dataRoot);
|
||||
const runChild = vi.fn();
|
||||
const service = new WorkspacePreprocessingService({
|
||||
dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(baseWorkspace),
|
||||
runChild,
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
const result = await service({ repository: catalog, runChild }).run({
|
||||
workspaceId: "catalog-workspace",
|
||||
});
|
||||
|
||||
const result = await service.indexSchema({ workspaceId: "psd-clinical" });
|
||||
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
|
||||
expect(runChild).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("filesystem Evidence proceeds after materialization and private HTTP hosts outside the allowlist are refused", async () => {
|
||||
const filesystem = fixture(filesystemWorkspace);
|
||||
filesystem.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1 } }),
|
||||
stderr: "",
|
||||
});
|
||||
const materialized = await filesystem.service.preprocessEvidence({ workspaceId: "fs-workspace" });
|
||||
expect(materialized).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
expect(filesystem.runChild).toHaveBeenCalledTimes(1);
|
||||
|
||||
const httpDataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
|
||||
roots.push(httpDataRoot);
|
||||
const httpService = new WorkspacePreprocessingService({
|
||||
dataRoot: httpDataRoot,
|
||||
acquireActiveRuntime: async () => runtime(privateHttpWorkspace, "http-workspace"),
|
||||
runChild: vi.fn(),
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: true }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
httpPrivateHostAllowlist: ["metadata.internal"],
|
||||
});
|
||||
|
||||
const refused = await httpService.preprocessEvidence({ workspaceId: "http-workspace" });
|
||||
expect(refused).toMatchObject({ status: "failed", code: "egress_policy_refused" });
|
||||
});
|
||||
|
||||
test("full runs follow the explicit order and finish unchanged when no Evidence source exists", async () => {
|
||||
const f = fixture();
|
||||
f.runChild
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
|
||||
stderr: "",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 0,
|
||||
candidate_digest: "sha256:" + "0".repeat(64),
|
||||
candidate_yaml: "tables: []\n",
|
||||
}),
|
||||
stderr: "",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 },
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const result = await f.service.run({ workspaceId: "psd-clinical" });
|
||||
expect(result).toMatchObject({
|
||||
status: "succeeded",
|
||||
code: "ok",
|
||||
completedStages: ["dwh", "fk_suggest", "schema_index"],
|
||||
warnings: ["workspace has no Evidence source"],
|
||||
completedStages: ["catalog_snapshot", "catalog_metadata", "lsh", "schema_index"],
|
||||
});
|
||||
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv.slice(0, 2).join(" "))).toEqual([
|
||||
"preprocess dwh",
|
||||
"schema suggest-fks",
|
||||
"vector index-schema",
|
||||
]);
|
||||
expect(runChild).toHaveBeenCalledWith(expect.objectContaining({
|
||||
argv: [
|
||||
"preprocess", "catalog", "--catalog-metadata",
|
||||
expect.stringMatching(/\/catalog-metadata\.json$/),
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
],
|
||||
}));
|
||||
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
|
||||
"catalog-workspace",
|
||||
9,
|
||||
"sha256:" + "f".repeat(64),
|
||||
{ status: "succeeded" },
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("schema accept validates the synced Git blob and records the review", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: {}\n",
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
|
||||
const runId = suggest.runId!;
|
||||
const candidateDigest = suggest.artifactIdentities![0]!.digest;
|
||||
const synced = syncAnnotations({
|
||||
dataRoot: f.dataRoot,
|
||||
workspaceId: "psd-clinical",
|
||||
commit: "a".repeat(40),
|
||||
blobId: "b".repeat(40),
|
||||
contents: Buffer.from("tables: {}\n"),
|
||||
});
|
||||
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
orphan_count: 0,
|
||||
annotations_digest: synced.contentDigest,
|
||||
reviewed_candidates_digest: candidateDigest,
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const result = await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
|
||||
expect(result).toMatchObject({ status: "succeeded", code: "ok", operation: "schema accept" });
|
||||
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
|
||||
expect(state.readFkReview(runId)).toMatchObject({
|
||||
reviewedCandidatesDigest: candidateDigest,
|
||||
annotationsDigest: synced.contentDigest,
|
||||
workspaceRevision: "a".repeat(40),
|
||||
blobId: "b".repeat(40),
|
||||
});
|
||||
test("preprocessing fails closed when the PostgreSQL Catalog is unavailable", async () => {
|
||||
const result = await service().run({ workspaceId: "catalog-workspace" });
|
||||
expect(result).toMatchObject({ status: "failed", code: "catalog_not_ready" });
|
||||
});
|
||||
|
||||
test("schema accept fails closed without --yes, for an unknown run, or with no synced annotations", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: {}\n",
|
||||
}),
|
||||
stderr: "",
|
||||
test("preprocessing refuses a concurrent Catalog run without touching derived data", async () => {
|
||||
const catalog = repository({
|
||||
beginPreprocessing: vi.fn(async () => ({ kind: "already_running" as const })),
|
||||
});
|
||||
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
|
||||
const runId = suggest.runId!;
|
||||
const runChild = vi.fn();
|
||||
|
||||
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: false }))
|
||||
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
const result = await service({ repository: catalog, runChild }).run({
|
||||
workspaceId: "catalog-workspace",
|
||||
});
|
||||
|
||||
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId: "e".repeat(32), yes: true }))
|
||||
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
|
||||
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true }))
|
||||
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
expect(f.runChild).toHaveBeenCalledTimes(1);
|
||||
expect(result).toMatchObject({ status: "failed", code: "preprocessing_conflict" });
|
||||
expect(runChild).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("full runs continue after schema accept only when the accepted blob matches the current revision", async () => {
|
||||
const f = fixture();
|
||||
const revision = "a".repeat(40);
|
||||
f.runChild
|
||||
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const blocked = await f.service.run({ workspaceId: "psd-clinical" });
|
||||
expect(blocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
|
||||
const runId = blocked.runId!;
|
||||
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
|
||||
const candidateDigest = state.readFkCandidates(runId)!.digest;
|
||||
|
||||
const synced = syncAnnotations({
|
||||
dataRoot: f.dataRoot,
|
||||
workspaceId: "psd-clinical",
|
||||
commit: revision,
|
||||
blobId: "b".repeat(40),
|
||||
contents: Buffer.from("tables: {}\n"),
|
||||
test("preprocessing records a failed PostgreSQL state when its hidden worker fails", async () => {
|
||||
const catalog = repository();
|
||||
const instance = service({
|
||||
repository: catalog,
|
||||
runChild: async () => ({ exitCode: 1, stdout: "", stderr: "private failure" }),
|
||||
});
|
||||
|
||||
// Accept writes the review; the resume then passes the gate and reaches schema indexing.
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", orphan_count: 0, annotations_digest: synced.contentDigest, reviewed_candidates_digest: candidateDigest }),
|
||||
stderr: "",
|
||||
});
|
||||
await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
|
||||
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 } }),
|
||||
stderr: "",
|
||||
});
|
||||
const resumed = await f.service.run({ workspaceId: "psd-clinical", resumeRunId: runId });
|
||||
expect(resumed).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
|
||||
// A review whose accepted blob digest no longer matches the current revision stays blocked.
|
||||
const second = fixture();
|
||||
second.runChild
|
||||
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
|
||||
stderr: "",
|
||||
});
|
||||
const secondBlocked = await second.service.run({ workspaceId: "psd-clinical" });
|
||||
const secondRunId = secondBlocked.runId!;
|
||||
const secondState = new PreprocessingStateStore({ dataRoot: second.dataRoot, workspaceId: "psd-clinical" });
|
||||
const secondCandidate = secondState.readFkCandidates(secondRunId)!.digest;
|
||||
secondState.writeFkReview(secondRunId, {
|
||||
reviewedCandidatesDigest: secondCandidate,
|
||||
annotationsDigest: "sha256:" + "0".repeat(64),
|
||||
workspaceRevision: revision,
|
||||
blobId: "b".repeat(40),
|
||||
});
|
||||
const stillBlocked = await second.service.run({ workspaceId: "psd-clinical", resumeRunId: secondRunId });
|
||||
expect(stillBlocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
|
||||
await expect(instance.run({ workspaceId: "catalog-workspace" })).rejects.toThrow(
|
||||
"workspace child failed",
|
||||
);
|
||||
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
|
||||
"catalog-workspace",
|
||||
9,
|
||||
"sha256:" + "f".repeat(64),
|
||||
{ status: "failed", errorCode: "schema_index_failed" },
|
||||
);
|
||||
});
|
||||
|
||||
test("vector rebuild recreates the full collection contract including keyword indexes", async () => {
|
||||
const f = fixture();
|
||||
// mock fetch: DELETE ok, then reconcileCollection self-heals create + indexes (real fetch in deps)
|
||||
const calls: string[] = [];
|
||||
const fakeFetch = async (url: string, init?: any) => {
|
||||
calls.push(`${init?.method ?? "GET"} ${url}`);
|
||||
if ((init?.method ?? "GET") === "DELETE") return new Response("", { status: 200 });
|
||||
if (url.endsWith("/collections/psd-clinical") && init?.method === "PUT") return new Response("", { status: 200 });
|
||||
if (url.endsWith("/collections/psd-clinical") && init?.method === "GET") {
|
||||
return new Response(JSON.stringify({ result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, payload_schema: { content_hash: { data_type: "keyword" }, document_id: { data_type: "keyword" }, kind: { data_type: "keyword" }, record_key: { data_type: "keyword" }, record_kind: { data_type: "keyword" }, vector_generation: { data_type: "keyword" }, workspace_id: { data_type: "keyword" }, workspace_revision: { data_type: "keyword" } } } }), { status: 200 });
|
||||
}
|
||||
if (url.endsWith("/collections/psd-clinical/index") && init?.method === "PUT") return new Response("", { status: 200 });
|
||||
return new Response(JSON.stringify({ result: {} }), { status: 200 });
|
||||
};
|
||||
const service = new WorkspacePreprocessingService({
|
||||
dataRoot: f.dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(baseWorkspace),
|
||||
runChild: vi.fn(),
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: true }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
test("semantic preflight failure is persisted before returning", async () => {
|
||||
const catalog = repository();
|
||||
const result = await service({
|
||||
repository: catalog,
|
||||
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
|
||||
}).run({ workspaceId: "catalog-workspace" });
|
||||
|
||||
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
|
||||
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
|
||||
"catalog-workspace",
|
||||
9,
|
||||
"sha256:" + "f".repeat(64),
|
||||
{ status: "failed", errorCode: "semantic_index_incompatible" },
|
||||
);
|
||||
});
|
||||
|
||||
test.each(["refresh", "decide"] as const)("Evidence %s calls only the source worker and preserves Catalog readiness", async action => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({exitCode: 0, stdout: JSON.stringify({status: "succeeded", counts: {changed: 1}}), stderr: ""}));
|
||||
const result = await service({repository: catalog, runChild}).evidenceSources({workspaceId: "catalog-workspace", action,
|
||||
...(action === "decide" ? {sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "replace" as const} : {}), actor: "Curator"});
|
||||
expect(result.status).toBe("succeeded");
|
||||
expect(runChild).toHaveBeenCalledWith({configPath: expect.any(String), argv: expect.arrayContaining(["evidence", "sources", action, "-c", "/dev/fd/3", "--actor", "Curator"])});
|
||||
expect(catalog.beginPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.clearPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("failed source activation reports the saved decision for retry", async () => {
|
||||
const result = await service({runChild: async () => ({exitCode: 1, stdout: JSON.stringify({status: "failed", saved: true, error: "Decision saved; retry indexing"}), stderr: "secret"})})
|
||||
.evidenceSources({workspaceId: "catalog-workspace", action: "decide", sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "keep"});
|
||||
expect(result).toMatchObject({status: "failed", warnings: ["Decision saved; retry indexing"]});
|
||||
});
|
||||
|
||||
test("clear invalidates Catalog readiness before clearing only derived worker data", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ counts: { referenceCollections: 1, derivedPaths: 3 } }),
|
||||
stderr: "",
|
||||
}));
|
||||
|
||||
const result = await service({ repository: catalog, runChild }).clear({
|
||||
workspaceId: "catalog-workspace",
|
||||
});
|
||||
// replace global fetch used by vectorRebuild/reconcileCollection
|
||||
const original = globalThis.fetch;
|
||||
globalThis.fetch = fakeFetch as any;
|
||||
try {
|
||||
const result = await service.vectorRebuild({ workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true });
|
||||
expect(result).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
} finally {
|
||||
globalThis.fetch = original;
|
||||
}
|
||||
expect(calls.some((c) => c.startsWith("DELETE "))).toBe(true);
|
||||
|
||||
expect(catalog.clearPreprocessing).toHaveBeenCalledWith("catalog-workspace");
|
||||
expect(runChild).toHaveBeenCalledWith({
|
||||
argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
|
||||
configPath: expect.any(String),
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
status: "succeeded",
|
||||
code: "ok",
|
||||
operation: "preprocess clear",
|
||||
counts: { referenceCollections: 1, derivedPaths: 3 },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -47,6 +47,15 @@ test("job state creates durable 0600 JSON and enforces same-revision resume", as
|
||||
embeddingId: "ollama/qwen3-embedding:0.6b",
|
||||
embeddingDimensions: 1024,
|
||||
});
|
||||
expect(store.readLatestJob()).toMatchObject({
|
||||
runId: job.runId,
|
||||
status: "active",
|
||||
completedStages: [],
|
||||
});
|
||||
|
||||
job.completedStages.push("catalog_snapshot");
|
||||
store.writeJob(job);
|
||||
expect(store.readLatestJob()?.completedStages).toEqual(["catalog_snapshot"]);
|
||||
|
||||
await expect(store.beginJob({
|
||||
operation: "preprocess dwh",
|
||||
|
||||
@@ -19,11 +19,6 @@ const validYaml = `workspace:
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
|
||||
@@ -34,119 +29,12 @@ function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
|
||||
`);
|
||||
}
|
||||
|
||||
function withDwhRestTransport(source: string): string {
|
||||
return source.replace(
|
||||
"supported_transports: [postgres_direct]",
|
||||
"supported_transports: [postgres_direct, rest_api]",
|
||||
);
|
||||
}
|
||||
|
||||
function withDwhRestDiagnostic(source: string): string {
|
||||
return withDwhRestTransport(source).concat(`diagnostics:
|
||||
dwh_rest:
|
||||
method: GET
|
||||
path: /health
|
||||
auth: none
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
`);
|
||||
}
|
||||
|
||||
function withEmbeddingDiagnostic(source: string): string {
|
||||
return source.concat(`diagnostics:
|
||||
embedding:
|
||||
method: GET
|
||||
path: /models
|
||||
auth: none
|
||||
response:
|
||||
model: model
|
||||
dimensions: dimensions
|
||||
`);
|
||||
}
|
||||
|
||||
function withDwhRestAndEmbeddingDiagnostics(source: string): string {
|
||||
return withDwhRestTransport(source).concat(`diagnostics:
|
||||
dwh_rest:
|
||||
method: GET
|
||||
path: /health
|
||||
auth: none
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
embedding:
|
||||
method: GET
|
||||
path: /models
|
||||
auth: none
|
||||
response:
|
||||
model: model
|
||||
dimensions: dimensions
|
||||
`);
|
||||
}
|
||||
|
||||
function withVectorRestTransport(source: string): string {
|
||||
return source.replace(
|
||||
"supported_transports: [pgvector_direct]",
|
||||
"supported_transports: [pgvector_direct, rest_api]",
|
||||
);
|
||||
}
|
||||
|
||||
function withVectorMetadataDiagnostic(source: string): string {
|
||||
return withVectorRestTransport(source).concat(`diagnostics:
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: none
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
`);
|
||||
}
|
||||
|
||||
function withReversibleVectorProbe(source: string): string {
|
||||
return withVectorRestTransport(source).concat(`diagnostics:
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: none
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
reversible_probe:
|
||||
method: POST
|
||||
path: /probe
|
||||
auth: bearer
|
||||
response:
|
||||
operation: operation
|
||||
`);
|
||||
}
|
||||
|
||||
function legacyV1Yaml(source = validYaml): string {
|
||||
return source
|
||||
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
|
||||
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace(" provider: ollama_internal", " provider: ollama_compatible")
|
||||
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
|
||||
.replace("schema_version: 4", "schema_version: 1");
|
||||
return source.replace("schema_version: 4", "schema_version: 1");
|
||||
}
|
||||
|
||||
function legacyV2Yaml(source = validYaml): string {
|
||||
return source
|
||||
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
|
||||
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace(" provider: ollama_internal", " provider: ollama_compatible")
|
||||
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
|
||||
.replace("schema_version: 4", "schema_version: 2");
|
||||
return source.replace("schema_version: 4", "schema_version: 2");
|
||||
}
|
||||
|
||||
const runFile = promisify(execFile);
|
||||
@@ -605,23 +493,6 @@ test("keeps content-only historical descriptor revisions distinguishable by comm
|
||||
expect(oldPinned.workspace).toEqual(newPinned.workspace);
|
||||
});
|
||||
|
||||
test.each([
|
||||
["adds", validYaml, withDwhRestDiagnostic(validYaml)],
|
||||
["removes", withDwhRestDiagnostic(validYaml), validYaml],
|
||||
])("pulls a curator change that %s a diagnostics branch without API rewrite", async (_operation, baseSource, remoteSource) => {
|
||||
const remote = await fixture(baseSource);
|
||||
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
||||
await registry.bootstrap();
|
||||
const initial = await registry.read("psd-clinical");
|
||||
writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), remoteSource);
|
||||
await git(remote.source, ["add", "psd-clinical/workspace.yaml"]);
|
||||
await git(remote.source, ["commit", "-m", `Registry ${_operation} diagnostic branch`]);
|
||||
await git(remote.source, ["push", "origin", "main"]);
|
||||
|
||||
await registry.pull();
|
||||
const updated = await registry.read("psd-clinical");
|
||||
expect(updated.revision.commit).not.toBe(initial.revision.commit);
|
||||
});
|
||||
test.each([
|
||||
["v1", legacyV1Yaml()],
|
||||
["v2", legacyV2Yaml()],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
mkdirSync,
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
renderActiveWorkspaceRuntime,
|
||||
renderWorkspaceRuntimeFromSnapshotPath,
|
||||
} from "../src/workspaces/runtime-config-lease.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const runFile = promisify(execFile);
|
||||
const roots: string[] = [];
|
||||
@@ -70,11 +71,6 @@ workspaces: [{id: psd-clinical, name: Runtime Lease}]
|
||||
id: psd-clinical
|
||||
name: Runtime Lease
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: filesystem
|
||||
@@ -88,9 +84,6 @@ evidence:
|
||||
await git(source, ["push", "origin", "main"]);
|
||||
|
||||
mkdirSync(secretRoot);
|
||||
const passwordFile = join(secretRoot, "dwh-password");
|
||||
writeFileSync(passwordFile, "secret", { mode: 0o600 });
|
||||
chmodSync(passwordFile, 0o600);
|
||||
mkdirSync(dataRoot);
|
||||
|
||||
const registryConfig: WorkspaceRegistryConfig = {
|
||||
@@ -107,12 +100,31 @@ evidence:
|
||||
const registry = new WorkspaceRegistry(registryConfig);
|
||||
await registry.bootstrap();
|
||||
const revision = (await registry.list())[0];
|
||||
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse.internal");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "reader");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE", passwordFile);
|
||||
const workspaceSecretStore = new WorkspaceSecretStore({
|
||||
root: join(root, "vault"),
|
||||
runtimeRoot: join(root, "runtime-secrets"),
|
||||
installationId: "test",
|
||||
});
|
||||
workspaceSecretStore.putMany("psd-clinical", { "catalog.dwh.password": "secret" });
|
||||
const catalogDatabase = {
|
||||
id: "database-1",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres" as const,
|
||||
databaseName: "analytics",
|
||||
schema: "mart",
|
||||
binding: {
|
||||
transport: "postgres_direct" as const,
|
||||
host: "warehouse.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
version: 1,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
connectionStatus: "reachable" as const,
|
||||
metadataContentRevision: 1,
|
||||
preprocessingStatus: "failed" as const,
|
||||
};
|
||||
|
||||
return {
|
||||
dataRoot,
|
||||
@@ -121,6 +133,8 @@ evidence:
|
||||
registry,
|
||||
registryConfig,
|
||||
revision,
|
||||
workspaceSecretStore,
|
||||
catalogDatabase,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -140,6 +154,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const active = await renderActiveWorkspaceRuntime({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -150,6 +166,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
expect(active.renderedConfig).toBe(direct.renderedConfig);
|
||||
@@ -158,27 +176,6 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
||||
expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
test("renders a revision-qualified annotations root for the active revision", async () => {
|
||||
const f = await fixture();
|
||||
const active = await renderActiveWorkspaceRuntime({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
registryConfig: f.registryConfig,
|
||||
harnessDir: f.harnessDir,
|
||||
configPath: "config/tht.yaml",
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
});
|
||||
const rendered = parse(active.renderedConfig) as Record<string, any>;
|
||||
|
||||
expect(rendered.paths.annotations_root).toBe(
|
||||
join(f.dataRoot, "sessions", "psd-clinical", "revisions", f.revision.commit, "artifacts"),
|
||||
);
|
||||
expect(rendered.roots.annotations_root).toBe(rendered.paths.annotations_root);
|
||||
expect(rendered.paths.artifacts).toBe(join(f.dataRoot, "sessions", "psd-clinical", "artifacts"));
|
||||
});
|
||||
|
||||
test("deterministic operator leases are keyed by logical identity and stable across calls", async () => {
|
||||
const f = await fixture();
|
||||
const first = await publishDeterministicRuntimeConfigLease({
|
||||
@@ -190,6 +187,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const second = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -200,9 +199,11 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
const suffix = first.inputFingerprint.slice(7, 23);
|
||||
const suffix = createHash("sha256").update(first.inputFingerprint + "\n" + readFileSync(first.path, "utf8")).digest("hex").slice(0, 16);
|
||||
expect(second.path).toBe(first.path);
|
||||
expect(first.path).toBe(join(
|
||||
f.dataRoot,
|
||||
@@ -236,7 +237,11 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
path: first.path,
|
||||
});
|
||||
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal");
|
||||
const changedDatabase = {
|
||||
...f.catalogDatabase,
|
||||
binding: { ...f.catalogDatabase.binding, host: "warehouse-two.internal" },
|
||||
version: 2,
|
||||
};
|
||||
const changed = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
@@ -246,6 +251,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: changedDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
expect(changed.path).not.toBe(first.path);
|
||||
expect(changed.inputFingerprint).not.toBe(first.inputFingerprint);
|
||||
@@ -267,6 +274,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
})).toThrow(/trusted runtime snapshot/i);
|
||||
expect(() => renderWorkspaceRuntimeFromSnapshotPath({
|
||||
snapshotPath: symlink,
|
||||
@@ -275,6 +284,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
})).toThrow(/trusted runtime snapshot/i);
|
||||
});
|
||||
|
||||
@@ -288,6 +299,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const lease = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -298,6 +311,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
const sessionCanonical = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(parse(session.renderedConfig)));
|
||||
@@ -319,6 +334,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const firstIdentity = firstLease.effectiveConfigIdentity;
|
||||
|
||||
@@ -341,6 +358,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
expect(secondLease.workspaceRevision).toBe(current.commit);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync,
|
||||
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -25,11 +25,6 @@ const canonicalWorkspace = `workspace:
|
||||
id: psd-clinical
|
||||
name: Runtime handoff
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
const filesystemWorkspace = `${canonicalWorkspace}evidence:
|
||||
@@ -106,6 +101,38 @@ async function fixture(workspaceSource = filesystemWorkspace) {
|
||||
const registry = new WorkspaceRegistry(registryConfig);
|
||||
await registry.bootstrap();
|
||||
const revision = (await registry.list())[0];
|
||||
const workspaceSecretStore = new WorkspaceSecretStore({
|
||||
root: join(root, "workspace-secrets"),
|
||||
runtimeRoot: join(root, "workspace-secret-runtime"),
|
||||
installationId: "test",
|
||||
});
|
||||
workspaceSecretStore.putMany("psd-clinical", {
|
||||
"catalog.dwh.password": "dwh-password-value",
|
||||
"evidence.signed_urls": secretContents["evidence-signed-urls.json"],
|
||||
"evidence.access_key": secretContents["evidence-access"],
|
||||
"evidence.secret_key": secretContents["evidence-secret"],
|
||||
"evidence.session_token": secretContents["evidence-token"],
|
||||
});
|
||||
const catalogDatabase = {
|
||||
id: "database-1",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres" as const,
|
||||
databaseName: "analytics",
|
||||
schema: "mart",
|
||||
binding: {
|
||||
transport: "postgres_direct" as const,
|
||||
host: "dwh.invalid",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
version: 1,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
connectionStatus: "reachable" as const,
|
||||
metadataContentRevision: 1,
|
||||
preprocessingStatus: "failed" as const,
|
||||
};
|
||||
const catalogRepository = { getByWorkspace: async () => catalogDatabase } as any;
|
||||
const environment = {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
|
||||
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.invalid",
|
||||
@@ -119,7 +146,10 @@ async function fixture(workspaceSource = filesystemWorkspace) {
|
||||
};
|
||||
for (const [name, value] of Object.entries(environment)) vi.stubEnv(name, value);
|
||||
vi.stubEnv("THT_HOME", join(root, "home"));
|
||||
return { root, source, dataRoot, secretRoot, registry, registryConfig, revision };
|
||||
return {
|
||||
root, source, dataRoot, secretRoot, registry, registryConfig, revision,
|
||||
workspaceSecretStore, catalogDatabase, catalogRepository,
|
||||
};
|
||||
}
|
||||
|
||||
function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
|
||||
@@ -130,6 +160,8 @@ function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
|
||||
dataRoot: f.dataRoot,
|
||||
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
catalogRepository: f.catalogRepository,
|
||||
} as any);
|
||||
}
|
||||
|
||||
@@ -165,7 +197,7 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
|
||||
runtimeRoot,
|
||||
installationId: "test",
|
||||
});
|
||||
secretStore.put("psd-clinical", "dwh.password", "vault-runtime-password");
|
||||
secretStore.put("psd-clinical", "catalog.dwh.password", "vault-runtime-password");
|
||||
const runner = new ThtRunner({
|
||||
thtBin,
|
||||
harnessDir,
|
||||
@@ -174,9 +206,10 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
|
||||
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
workspaceSecretStore: secretStore,
|
||||
catalogRepository: f.catalogRepository,
|
||||
} as any);
|
||||
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const rendered = parse(readFileSync(lease.path, "utf8")) as {
|
||||
database: { password_file: string };
|
||||
};
|
||||
@@ -185,30 +218,11 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
|
||||
expect(existsSync(rendered.database.password_file)).toBe(false);
|
||||
});
|
||||
|
||||
test("ThtRunner binds and cleans the effective relationship snapshot with its runtime lease", async () => {
|
||||
const f = await fixture();
|
||||
const runner = runnerFor(f);
|
||||
const relationships = JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
workspaceId: "psd-clinical",
|
||||
relationships: [],
|
||||
});
|
||||
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath, relationships);
|
||||
const rendered = parse(readFileSync(lease.path, "utf8")) as {
|
||||
paths: { effective_relationships: string };
|
||||
};
|
||||
|
||||
expect(readFileSync(rendered.paths.effective_relationships, "utf8")).toBe(relationships);
|
||||
lease.release();
|
||||
expect(existsSync(rendered.paths.effective_relationships)).toBe(false);
|
||||
});
|
||||
|
||||
test("separate runtime leases hand off byte-identical revision Evidence configs accepted by tht", async () => {
|
||||
const f = await fixture();
|
||||
const runner = runnerFor(f);
|
||||
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const second = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const second = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const expectedRoot = join(
|
||||
f.registryConfig.root,
|
||||
"snapshots",
|
||||
@@ -228,6 +242,7 @@ test("separate runtime leases hand off byte-identical revision Evidence configs
|
||||
source_identity: "workspace://psd-clinical",
|
||||
});
|
||||
expect(parse(firstYaml).evidence).toEqual({
|
||||
local_archive_root: join(f.registryConfig.root, "repo", "psd-clinical"),
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: expectedRoot,
|
||||
@@ -263,7 +278,7 @@ test("separate runtime leases hand off byte-identical revision Evidence configs
|
||||
test("real Evidence-content-only commit changes runtime identity and root with identical descriptor YAML", async () => {
|
||||
const f = await fixture();
|
||||
const runner = runnerFor(f);
|
||||
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const descriptorBefore = readFileSync(f.revision.snapshotPath, "utf8");
|
||||
writeFileSync(
|
||||
join(f.source, "psd-clinical", "evidence", "guide.md"),
|
||||
@@ -274,7 +289,7 @@ test("real Evidence-content-only commit changes runtime identity and root with i
|
||||
await git(f.source, ["push", "origin", "main"]);
|
||||
await f.registry.pull();
|
||||
const current = (await f.registry.list())[0];
|
||||
const second = runner.acquireWorkspaceRuntime(current.snapshotPath);
|
||||
const second = await runner.acquireWorkspaceRuntime(current.snapshotPath);
|
||||
|
||||
try {
|
||||
expect(current.commit).not.toBe(f.revision.commit);
|
||||
@@ -317,13 +332,13 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
|
||||
max_cache_bytes: 67890
|
||||
`));
|
||||
const runner = runnerFor(f);
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
try {
|
||||
const yaml = readFileSync(lease.path, "utf8");
|
||||
expect(parse(yaml).evidence.sources).toEqual([{
|
||||
type: "http",
|
||||
provenance_urls: ["https://evidence.example.test/guide.md"],
|
||||
signed_urls_file: realpathSync(join(f.secretRoot, "evidence-signed-urls.json")),
|
||||
signed_urls_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
connect_timeout: 1.25,
|
||||
read_timeout: 30.001,
|
||||
max_bytes: 12_345,
|
||||
@@ -343,7 +358,7 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
|
||||
}
|
||||
});
|
||||
|
||||
test("static S3 Evidence resolves only configured secret-root file paths", async () => {
|
||||
test("static S3 Evidence resolves only ephemeral vault materializations", async () => {
|
||||
const f = await fixture(evidenceWorkspace(` type: s3
|
||||
uri: s3://clinical-evidence/published/
|
||||
endpoint_url: https://s3.example.test/
|
||||
@@ -361,7 +376,7 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
|
||||
retain_published_generations: 7
|
||||
`));
|
||||
const runner = runnerFor(f);
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
try {
|
||||
const yaml = readFileSync(lease.path, "utf8");
|
||||
expect(parse(yaml).evidence.sources).toEqual([{
|
||||
@@ -370,9 +385,9 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
|
||||
prefix: "published/",
|
||||
endpoint_url: "https://s3.example.test/",
|
||||
region: "eu-west-1",
|
||||
access_key_file: realpathSync(join(f.secretRoot, "evidence-access")),
|
||||
secret_key_file: realpathSync(join(f.secretRoot, "evidence-secret")),
|
||||
session_token_file: realpathSync(join(f.secretRoot, "evidence-token")),
|
||||
access_key_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
secret_key_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
session_token_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
trusted_endpoint: true,
|
||||
allow_private_endpoint: true,
|
||||
allow_insecure_endpoint: false,
|
||||
|
||||
@@ -10,9 +10,9 @@ import {
|
||||
type SemanticRuntimeConfig,
|
||||
} from "../src/workspaces/runtime-renderer.js";
|
||||
import { supportsSessionRuntime } from "../src/workspaces/bindings.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV4 = parseWorkspaceYaml(`workspace:
|
||||
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -70,7 +70,14 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu
|
||||
},
|
||||
dwh: { type: "postgres_direct" },
|
||||
resources: {
|
||||
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
base_url: "http://qdrant:6333",
|
||||
collections: {
|
||||
reference: "psd-clinical-reference",
|
||||
memory: "psd-clinical-memory",
|
||||
},
|
||||
},
|
||||
embeddings: {
|
||||
provider: "ollama_internal", base_url: "http://embedding:11434",
|
||||
id: "ollama/qwen3-embedding:0.6b",
|
||||
@@ -133,7 +140,7 @@ function evidenceWorkspace(
|
||||
policy?: Record<string, unknown>,
|
||||
evidenceSchemaVersion?: number,
|
||||
) {
|
||||
return parseWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
|
||||
return parseRuntimeWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
|
||||
evidenceSchemaVersion === undefined ? "" : `\n schema_version: ${evidenceSchemaVersion}`
|
||||
}\n source: ${JSON.stringify(source)}${
|
||||
policy === undefined ? "" : `\n policy: ${JSON.stringify(policy)}`
|
||||
@@ -185,6 +192,7 @@ test("renders filesystem Evidence below the immutable revision content root with
|
||||
expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision);
|
||||
expect(rendered.evidence).toEqual({
|
||||
schema_version: 2,
|
||||
local_archive_root: "/srv/registry/repo/psd-clinical",
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`,
|
||||
@@ -196,7 +204,7 @@ test("renders filesystem Evidence below the immutable revision content root with
|
||||
max_chunk_chars: 4_000,
|
||||
retain_published_generations: 3,
|
||||
});
|
||||
expect(yaml).not.toContain("/srv/registry/repo");
|
||||
expect(rendered.evidence.sources[0].root).not.toContain("/srv/registry/repo");
|
||||
});
|
||||
|
||||
test("renders public HTTP Evidence with exact fractional-second timeouts and every policy limit", () => {
|
||||
@@ -216,6 +224,7 @@ test("renders public HTTP Evidence with exact fractional-second timeouts and eve
|
||||
}));
|
||||
|
||||
expect(rendered.evidence).toEqual({
|
||||
local_archive_root: "/srv/registry/repo/psd-clinical",
|
||||
sources: [{
|
||||
type: "http",
|
||||
urls: ["https://evidence.example.test/guide.md"],
|
||||
|
||||
@@ -8,12 +8,12 @@ import {
|
||||
resolveRuntimeBindingsWithWorkspaceSecrets,
|
||||
} from "../src/workspaces/secret-requirements.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
function workspace(extra = "") {
|
||||
return parseWorkspaceYaml(`workspace:
|
||||
return parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
|
||||
@@ -23,14 +23,11 @@ test("strict workspace v4 rejects model-bearing v3 descriptors", () => {
|
||||
expect(() => parseWorkspaceYaml(legacy)).toThrow();
|
||||
});
|
||||
|
||||
test("v3 to v4 migration removes only model/vector policy and bumps the version", () => {
|
||||
test("v3 to v4 migration removes database/model policy and bumps the version", () => {
|
||||
const migrated = migrateWorkspaceV3Yaml(legacy);
|
||||
const workspace = parseWorkspaceYaml(migrated);
|
||||
|
||||
expect(workspace.workspace).toMatchObject({ schema_version: 4, id: "abc" });
|
||||
expect(migrated).not.toMatch(/semantic_index|llm_policy/);
|
||||
expect(workspace.dwh).toEqual({
|
||||
engine: "postgres", database: "warehouse", schema: "public",
|
||||
supported_transports: ["postgres_direct"],
|
||||
});
|
||||
expect(workspace.dwh).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -7,9 +7,9 @@ import { afterEach, expect, test } from "vitest";
|
||||
import {
|
||||
resolveBinding, resolveEvidenceBinding, resolveRuntimeBindings, supportsSessionRuntime,
|
||||
} from "../src/workspaces/bindings.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV4 = parseWorkspaceYaml(`workspace:
|
||||
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -63,7 +63,7 @@ test("requires workspace-v4 REST credentials unless the DWH diagnostic declares
|
||||
THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
|
||||
}, []).missing).toContain("THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE");
|
||||
|
||||
const noAuth = parseWorkspaceYaml(`workspace:
|
||||
const noAuth = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: No auth
|
||||
@@ -89,7 +89,7 @@ diagnostics:
|
||||
test("rejects unsupported transports and secret paths outside configured roots", () => {
|
||||
const outside = secretPath("outside-password");
|
||||
const allowed = secretPath("allowed-password");
|
||||
const directOnly = parseWorkspaceYaml(`workspace:
|
||||
const directOnly = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Direct only
|
||||
@@ -130,7 +130,7 @@ test("runtime bindings contain only DWH and Evidence roles", () => {
|
||||
});
|
||||
|
||||
function withEvidence(source: Record<string, unknown>) {
|
||||
return parseWorkspaceYaml(`workspace:
|
||||
return parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { expect, test } from "vitest";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import {
|
||||
CATALOG_PATH,
|
||||
assertCatalogMatchesDescriptor,
|
||||
parseWorkspaceCatalogYaml,
|
||||
} from "../src/workspaces/catalog.js";
|
||||
|
||||
const descriptor = parseWorkspaceYaml(`workspace:
|
||||
const descriptor = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd
|
||||
name: Policlinico San Donato
|
||||
|
||||
@@ -3,9 +3,9 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { buildInstallationContract, renderWorkspaceDocs } from "../src/workspaces/contracts.js";
|
||||
import { type CanonicalWorkspace, parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { type CanonicalWorkspace, parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV4 = parseWorkspaceYaml(`workspace:
|
||||
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -38,7 +38,7 @@ test.each([
|
||||
["rest_api", "BASE_URL", "HOST"],
|
||||
["ssh_tunnel", "SSH_PRIVATE_KEY_FILE", "BASE_URL"],
|
||||
] as const)("documents only the DWH fields for %s", (transport, included, excluded) => {
|
||||
const descriptor = parseWorkspaceYaml(renderWorkspaceWithoutEvidence()
|
||||
const descriptor = parseRuntimeWorkspaceYaml(renderWorkspaceWithoutEvidence()
|
||||
.replace("[postgres_direct]", `[${transport}]`));
|
||||
const variables = buildInstallationContract(descriptor).variables;
|
||||
expect(variables.find(({ suffix }) => suffix === included)?.transports).toEqual([transport]);
|
||||
@@ -87,7 +87,7 @@ test.each([
|
||||
],
|
||||
},
|
||||
])("generates source-specific $mode Evidence file bindings", ({ source, expected }) => {
|
||||
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
const contract = buildInstallationContract(descriptor);
|
||||
const evidence = contract.variables.filter((variable) => variable.role === "EVIDENCE");
|
||||
|
||||
@@ -102,7 +102,7 @@ test.each([
|
||||
{ type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" },
|
||||
{ type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" },
|
||||
])("omits Evidence installation variables for $type modes without file credentials", (source) => {
|
||||
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
expect(buildInstallationContract(descriptor).variables.some((variable) => variable.role === "EVIDENCE"))
|
||||
.toBe(false);
|
||||
});
|
||||
@@ -151,7 +151,7 @@ const evidenceSources = [
|
||||
] as const;
|
||||
|
||||
test.each(evidenceSources)("renders deterministic public Evidence docs for $label", ({ source, variables }) => {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
|
||||
);
|
||||
const firstContract = buildInstallationContract(descriptor);
|
||||
@@ -178,7 +178,7 @@ test.each(evidenceSources)("renders deterministic public Evidence docs for $labe
|
||||
});
|
||||
|
||||
test("documents the S3 session token file as optional", () => {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:
|
||||
source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files }
|
||||
`,
|
||||
@@ -197,7 +197,7 @@ test("documents the S3 session token file as optional", () => {
|
||||
});
|
||||
|
||||
test("documents same-revision filesystem ownership without claiming P1 materialization", () => {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:\n source: { type: filesystem, uri: psd-clinical/evidence }\n`,
|
||||
);
|
||||
const docs = renderWorkspaceDocs(descriptor).markdown;
|
||||
@@ -236,7 +236,7 @@ test.each([
|
||||
const previous = process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE;
|
||||
process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = binding;
|
||||
try {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
|
||||
);
|
||||
const generated = JSON.stringify({
|
||||
|
||||
@@ -9,9 +9,9 @@ import {
|
||||
type DiagnosticAdapters,
|
||||
} from "../src/workspaces/diagnostics.js";
|
||||
import type { RuntimeBindings } from "../src/workspaces/runtime-renderer.js";
|
||||
import { parseWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspace = parseWorkspaceYaml(`workspace:
|
||||
const workspace = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -81,9 +81,11 @@ test("diagnoses workspace-v4 DWH plus installation-derived Qdrant and Ollama", a
|
||||
role: "dwh", transport: "postgres_direct",
|
||||
resource: { database: "warehouse", schema: "datawarehouse" },
|
||||
}));
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({
|
||||
baseUrl: "http://qdrant:6333", collection: "psd-clinical",
|
||||
}));
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
|
||||
expect(vi.mocked(adapters.inspectQdrant).mock.calls.map(([request]) => request.collection)).toEqual([
|
||||
"psd-clinical-reference",
|
||||
"psd-clinical-memory",
|
||||
]);
|
||||
expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({
|
||||
baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b",
|
||||
}));
|
||||
@@ -107,14 +109,14 @@ test("can delegate the DWH probe to Database Management", async () => {
|
||||
}],
|
||||
});
|
||||
expect(adapters.probeConnector).not.toHaveBeenCalled();
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(1);
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
|
||||
expect(adapters.probeEmbedding).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("reports incompatible internal Qdrant or Ollama metadata", async () => {
|
||||
const vector = await diagnose(successfulAdapters({
|
||||
inspectQdrant: vi.fn(async () => ({
|
||||
collection: "psd-clinical", dimensions: 768, distance: "cosine",
|
||||
inspectQdrant: vi.fn(async (request) => ({
|
||||
collection: request.collection, dimensions: 768, distance: "cosine",
|
||||
})),
|
||||
}))(workspace, bindings, { writeProbe: false });
|
||||
expect(vector.activatable).toBe(false);
|
||||
@@ -163,7 +165,7 @@ test("keeps workspace-v4 DWH SSH diagnostic-only and runtime-inactive", async ()
|
||||
});
|
||||
|
||||
test("uses the workspace-v4 declared DWH REST diagnostic and auth policy", async () => {
|
||||
const restWorkspace = parseWorkspaceYaml(`workspace:
|
||||
const restWorkspace = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: REST workspace
|
||||
@@ -428,7 +430,7 @@ test("uses a REST secret only as a header and redacts it from failed diagnostics
|
||||
const credentialFile = join(root, "api-key");
|
||||
const canary = "CANARY-REST-AUTH-SECRET";
|
||||
await writeFile(credentialFile, canary);
|
||||
const restDescriptor = parseWorkspaceYaml(`workspace:
|
||||
const restDescriptor = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: REST auth
|
||||
@@ -548,6 +550,9 @@ test("returns observed normalized Qdrant distance for semantic mismatch classifi
|
||||
|
||||
test("classifies an observed non-cosine Qdrant distance as semantic incompatibility", async () => {
|
||||
const fetchMock = vi.fn()
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
|
||||
}), { status: 200 }))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
|
||||
}), { status: 200 }))
|
||||
|
||||
@@ -16,11 +16,6 @@ const validYaml = `workspace:
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
const runFile = promisify(execFile);
|
||||
|
||||
@@ -15,16 +15,6 @@ export const validYaml = `workspace:
|
||||
name: Policlinico San Donato
|
||||
description: Clinical data warehouse workspace
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
port: 5432
|
||||
timeout_ms: 5000
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
- rest_api
|
||||
- ssh_tunnel
|
||||
`;
|
||||
|
||||
test("rejects unknown keys and invalid immutable IDs", () => {
|
||||
@@ -41,21 +31,18 @@ test("rejects executable or otherwise custom YAML tags in canonical descriptors"
|
||||
))).toThrow(/tag|yaml/i);
|
||||
});
|
||||
|
||||
test("accepts optional connection ports and timeouts but rejects unsafe values", () => {
|
||||
expect(parseWorkspaceYaml(validYaml).dwh.port).toBe(5432);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 0")))
|
||||
.toThrow(/port/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 65536")))
|
||||
.toThrow(/port/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("timeout_ms: 5000", "timeout_ms: 0")))
|
||||
.toThrow(/timeout/i);
|
||||
test("rejects database configuration and diagnostics in authored workspace YAML", () => {
|
||||
expect(() => parseWorkspaceYaml(`${validYaml}dwh:\n engine: postgres\n database: d\n schema: s\n supported_transports: [postgres_direct]\n`))
|
||||
.toThrow(/must not contain database configuration/i);
|
||||
expect(() => parseWorkspaceYaml(`${validYaml}diagnostics:\n dwh_rest:\n method: GET\n path: \/health\n auth: none\n`))
|
||||
.toThrow(/must not contain database configuration/i);
|
||||
});
|
||||
|
||||
test("accepts a model-free schema v4 workspace", () => {
|
||||
test("accepts a database-free schema v4 workspace", () => {
|
||||
expect(parseWorkspaceYaml(validYaml)).toMatchObject({
|
||||
workspace: { schema_version: 4, id: "psd-clinical" },
|
||||
dwh: { database: "postgres", schema: "datawarehouse" },
|
||||
});
|
||||
expect(parseWorkspaceYaml(validYaml)).not.toHaveProperty("dwh");
|
||||
});
|
||||
|
||||
test("committed example descriptors parse as exact schema v4 workspaces", () => {
|
||||
@@ -137,7 +124,7 @@ llm_policy:
|
||||
- zai/glm-5.2
|
||||
`],
|
||||
])("rejects schema %s descriptors at parser and object-validator boundaries", (_version, yaml) => {
|
||||
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|4/i);
|
||||
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|database configuration|4/i);
|
||||
expect(() => validateWorkspaceDescriptor(parse(yaml))).toThrow(/schema_version|invalid literal|4/i);
|
||||
});
|
||||
|
||||
@@ -159,7 +146,7 @@ test("constructs diagnostic URLs only when the resolved URL remains on the servi
|
||||
)).toThrow(/origin/i);
|
||||
});
|
||||
|
||||
test("rejects REST diagnostic declarations without their matching connector transport", () => {
|
||||
test("rejects REST diagnostic declarations in authored descriptors", () => {
|
||||
const diagnostics = `diagnostics:
|
||||
dwh_rest:
|
||||
method: POST
|
||||
@@ -171,7 +158,7 @@ test("rejects REST diagnostic declarations without their matching connector tran
|
||||
`;
|
||||
const declared = `${validYaml}${diagnostics}`;
|
||||
|
||||
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", ""))).toThrow(/dwh_rest/i);
|
||||
expect(() => parseWorkspaceYaml(declared)).toThrow(/database configuration/i);
|
||||
});
|
||||
|
||||
test("serializes canonical YAML that parses back to the same workspace", () => {
|
||||
|
||||
@@ -138,6 +138,30 @@ test("projects aggregate no-Evidence and completed-stage outcomes without rerunn
|
||||
expect(deps.persistJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("complete preprocessing preflights BM25 before continuing with Evidence", async () => {
|
||||
const deps = dependencies();
|
||||
deps.evidencePreflight.mockResolvedValue({
|
||||
ok: false as const,
|
||||
code: "semantic_index_incompatible" as const,
|
||||
});
|
||||
const state = job({ completedStages: ["catalog_snapshot", "schema_index"] });
|
||||
|
||||
const result = await continueEvidencePreprocessing(
|
||||
{ evidence: filesystemEvidence, job: state },
|
||||
deps,
|
||||
);
|
||||
|
||||
expect(deps.evidencePreflight).toHaveBeenCalledOnce();
|
||||
expect(deps.runStage).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
status: "failed",
|
||||
code: "semantic_index_incompatible",
|
||||
runId: "a".repeat(32),
|
||||
childRuns: {},
|
||||
completedStages: ["catalog_snapshot", "schema_index"],
|
||||
});
|
||||
});
|
||||
|
||||
test("preserves the narrow standalone projection for an already completed Evidence stage", async () => {
|
||||
const deps = dependencies();
|
||||
|
||||
|
||||
+16
-2
@@ -13,6 +13,7 @@ services:
|
||||
THT_BIN: /opt/venv/bin/tht
|
||||
THT_DATA_ROOT: /data
|
||||
SETTINGS_FILE: /data/settings/settings.json
|
||||
THT_EVIDENCE_HOST_REGISTRY_ROOT: ${THT_WORKSPACE_REGISTRY_ROOT:-}
|
||||
THT_MAINTENANCE_FILE: /data/settings/maintenance.json
|
||||
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
|
||||
THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE}
|
||||
@@ -99,7 +100,7 @@ services:
|
||||
image: thothii-core:local
|
||||
profiles: [catalog-maintenance]
|
||||
pull_policy: never
|
||||
command: ["node", "/app/backend/dist/catalog/migrate.js"]
|
||||
command: ["bash", "/app/docker/catalog-migrate.sh"]
|
||||
environment:
|
||||
THT_CATALOG_DB_HOST: catalog-db
|
||||
THT_CATALOG_DB_PORT: "5432"
|
||||
@@ -136,6 +137,11 @@ services:
|
||||
THT_LLM_URL: ${THT_LLM_URL:-}
|
||||
THT_INTERNAL_QDRANT_URL: http://qdrant:6333
|
||||
THT_INTERNAL_EMBEDDING_URL: http://embedding:11434
|
||||
THT_CATALOG_DB_HOST: catalog-db
|
||||
THT_CATALOG_DB_PORT: "5432"
|
||||
THT_CATALOG_DB_NAME: thothii_catalog
|
||||
THT_CATALOG_RUNTIME_USER: thothii_catalog_runtime
|
||||
THT_CATALOG_RUNTIME_PASSWORD_FILE: /run/secrets/catalog_runtime_password
|
||||
HOME: /tmp/thoth
|
||||
AWS_ACCESS_KEY_ID: ""
|
||||
AWS_SECRET_ACCESS_KEY: ""
|
||||
@@ -148,7 +154,7 @@ services:
|
||||
- type: volume
|
||||
source: workspace-registry
|
||||
target: /data/workspace-registry
|
||||
read_only: true
|
||||
read_only: false
|
||||
- type: volume
|
||||
source: sessions
|
||||
target: /data/sessions
|
||||
@@ -158,6 +164,7 @@ services:
|
||||
secrets:
|
||||
- source: thothii_secrets
|
||||
target: thothii.secrets
|
||||
- catalog_runtime_password
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
@@ -168,6 +175,13 @@ services:
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: "no"
|
||||
depends_on:
|
||||
catalog-db:
|
||||
condition: service_healthy
|
||||
qdrant:
|
||||
condition: service_healthy
|
||||
embedding-model-init:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- thothii
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Optional local profile: expose the existing curator checkout on the installation host.
|
||||
# Copy the existing registry repo to this path before enabling the override. Snapshot/state
|
||||
# volumes remain unchanged. THT_EVIDENCE_HOST_REGISTRY_ROOT must be an absolute host path.
|
||||
services:
|
||||
core:
|
||||
environment:
|
||||
THT_EVIDENCE_HOST_REGISTRY_ROOT: ${THT_EVIDENCE_HOST_REGISTRY_ROOT:?set an absolute host registry path}
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${THT_EVIDENCE_HOST_REGISTRY_ROOT:?set an absolute host registry path}/repo
|
||||
target: /data/workspace-registry/repo
|
||||
bind:
|
||||
create_host_path: false
|
||||
workspace-maintenance:
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${THT_EVIDENCE_HOST_REGISTRY_ROOT:?set an absolute host registry path}/repo
|
||||
target: /data/workspace-registry/repo
|
||||
bind:
|
||||
create_host_path: false
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user