docs: record server release and hand off remaining acceptance checks

Reset the activity panel when starting a new question so the landing navigation is restored. Detect and persist the original question language, pass it through runtime and widget descriptors, and scope HITL controls to that language.

Validated with gate, session, backend and frontend tests, TypeScript checks, Ruff and strict docs build. Rebuilt and restarted local core/frontend; both healthy and serving HTTP successfully.

.dockerignore

@@ -17,6 +17,7 @@ frontend/vite.database-management-prototype.config.ts
 !deploy/env/*.env.example
 deploy/thothii.env
 deploy/secrets/
+deploy/psd/
 harness/workspaces/*.yaml
 !harness/workspaces/local.yaml
 !harness/workspaces/tht.example.yaml
@@ -29,3 +30,7 @@ coverage/
 data/
 sessions/
 workspace-registry/
+
+.tht/
+
+deploy/local/

.gitea/workflows/publish-docs.yml

@@ -7,6 +7,11 @@ on:
     paths:
       - "docs/**"
       - "mkdocs.yml"
+      - "scripts/build-docs.sh"
+      - "scripts/verify-public-docs.py"
+      - "scripts/test-verify-public-docs.py"
+      - "scripts/verify-auth-docs.py"
+      - "scripts/test-verify-auth-docs.py"
       - "docs/requirements.txt"
       - ".gitea/workflows/publish-docs.yml"
   workflow_dispatch:
@@ -34,14 +39,25 @@ jobs:
         with:
           python-version: "3.x"
           cache: pip
-          cache-dependency-path: docs/requirements.txt
+          cache-dependency-path: docs/requirements.lock
 
       - name: Install MkDocs dependencies
-        run: python -m pip install -r docs/requirements.txt
+        run: python -m pip install -r docs/requirements.lock
+
+      - name: Test public documentation boundary
+        run: python scripts/test-verify-public-docs.py
+
+      - name: Test current authentication documentation
+        run: |
+          python scripts/verify-auth-docs.py auth
+          python scripts/verify-auth-docs.py dwh
+          python scripts/test-verify-auth-docs.py auth
+          python scripts/test-verify-auth-docs.py dwh
 
       - name: Build documentation
-        # Some documented source files intentionally live outside docs/.
-        run: mkdocs build
+        run: |
+          mkdocs build --strict
+          python scripts/verify-public-docs.py
 
       - name: Publish generated site to the pages branch
         working-directory: site

.gitignore

@@ -35,6 +35,7 @@ config/ca-chain.pem
 
 # ThothII deployment configuration and secret values (keep only the README tracked)
 deploy/.env
+deploy/env/local.env
 deploy/compose.connector-secrets.local.yaml
 deploy/compose.psd-local.yaml
 deploy/workspaces/psd.yaml
@@ -45,6 +46,8 @@ deploy/secrets/*
 # Per-installation configuration generated by `tht setup` (examples stay tracked).
 deploy/*/thothii-installation.yaml
 deploy/*/operator.env
+deploy/*/auth/
+deploy/*/generated/
 deploy/*/secrets/*
 !deploy/*/secrets/.gitkeep
 !deploy/*/secrets/*.example

AGENTS.md

@@ -83,7 +83,8 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
   `SseHub` fans them out over SSE to the browser. The separate PostgreSQL catalog stores database
   metadata and sequential AI description-generation runs. Description generation samples the DWH
   through read-only connectors and calls a short-lived Python LiteLLM helper; it does not use Pi or
-  expose a public CLI command. App settings still live in `backend/data/settings.json`.
+  expose a public CLI command. Sessions, metadata generation, and embedding resolve models from the
+  generated Installation Model Catalog; `thothii-installation.yaml` is its only authored source.
 
 - **Human-in-the-loop gate contract.** The model proposes; a human reviewer decides at gates
   via widgets (`reviewer_select` = single pick — a chosen option carrying a `decision` payload
@@ -97,13 +98,24 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
 - **`tht`'s `-c`/`--config` is a PER-COMMAND option** — it must follow the subcommand, never
   precede it (`ThtRunner.buildArgv` enforces this; prepending caused live 500s).
 - **`--json` output must be pristine** (only valid JSON on stdout) — used as a machine contract.
-- **UI strings are English; document *content* stays the workspace language** (Italian for
-  `psd`) because it's the real data. Only chrome/labels are English.
-- **Workspaces** (`harness/workspaces/*.yaml`) set the DB target and **absolute**
-  `paths.sessions/artifacts/indexes` — for `psd` these point at a *separate, uncommitted* repo
-  (`tht-workspace-psd/`). Secrets live ONLY in `harness/.env` (gitignored).
-- **Settings are global** (`backend/data/settings.json`: workspace/provider/model/thinking);
-  the New-session form is question-only.
+- **Localization:** deterministic UI uses the EN/IT catalogs with English fallback;
+  model interaction uses the session manifest's immutable `interaction_language`.
+  Workspace content, SQL, and identifiers remain unchanged. For shell modes, portal
+  integration, or translations, read `docs/operations/shell-and-localization.md`.
+- **Server deployment:** for the coordinated ThothII/Omics upgrade, follow
+  `docs/operations/server-codex-handoff.md`; it supersedes earlier Omics delivery
+  instructions. Omics source integration uses GitHub with no repository relay prerequisite.
+- **Server identity:** for portal login/logout, proxy headers, or Omics deploy, read
+  `docs/install/authentication-upstream.md` before changing authentication. Omics
+  uses embedded/upstream, not a second ThothII OIDC login. Full/embedded rendering
+  is documented in `docs/architecture/application-shell.md`; release acceptance
+  is in `docs/testing/authentication-manual-acceptance.md`.
+- **Workspace schema v4** defines workspace identity and optional Evidence only. PostgreSQL Metadata
+  Catalog owns database identity, binding, schema, descriptions, sensitivity, and relationships;
+  embedding/model facts come from the installation catalog. The legacy `harness/workspaces/*.yaml` runtime snapshots still use
+  absolute session/artifact/index paths; secrets stay in `harness/.env` (gitignored).
+- **Settings are global** (`backend/data/settings.json`: workspace/thinking). Provider/model choices
+  are ephemeral canonical catalog selections pinned into the session manifest.
 - **Resume**: a resumable session re-enters at its last incomplete phase. The backend refuses
   resume with 409 when `finalized` or `archived`, and `PiProcessManager.spawnFor` must send
   `/riprendi-sessione <id>` (resume mode) vs `/nuova-domanda` (new) — sending the wrong prompt

CONTEXT.md

@@ -91,21 +91,79 @@ correzione successiva crea una nuova sessione derivata, collegata a quella prece
 dopo la finalizzazione. Non può modificare il ledger, gli artifact canonici o lo stato
 terminale della sessione.
 
+## Memory
+
+**Memory Module** — Il modulo che possiede le conoscenze ed esperienze curate per
+migliorare schema linking e generazione SQL di domande future. Le Memory appartengono
+a un workspace e rimangono distinte dalle Evidence.
+
+**Memory Card** — L'unità di contenuto gestibile del Memory Module, con identità,
+ambito di applicazione e provenienza. Il formato è allineato per analogia alle
+Evidence, senza implicare la stessa origine o lo stesso percorso di pubblicazione.
+
+**Reusable Memory** — Una Memory Card che esprime un chiarimento di dominio, una
+regola di costruzione SQL o un errore da evitare con motivo compreso e approvato.
+La sua validità è circoscritta a un ambito esplicito e non deriva dalla sola
+approvazione di una scelta occasionale in una domanda.
+
+**Solved Question** — Una Memory Card che conserva una domanda risolta con la
+relativa soluzione SQL e il contesto necessario a interpretarla. È un exemplar
+consultativo: i parametri e le scelte del caso non diventano regole generali.
+
+**Memory Graph** — L'insieme dei collegamenti espliciti fra card che contribuisce
+al recupero di conoscenze pertinenti oltre alla somiglianza del contenuto. Il
+ritrovamento di una card tramite un collegamento non ne implica l'approvazione.
+
+**Memory Link** — Un collegamento curato fra card, con destinazione e significato
+espliciti, che contribuisce alla consultazione di contenuti pertinenti. La sua
+rimozione non comporta la cancellazione delle card collegate.
+
 ## Evidence
 
+**Context specialist** — La persona competente sul dominio che redige e cura il
+contenuto delle Evidence. Può essere distinta da chi amministra l'installazione;
+il suo lavoro di redazione non richiede accesso al database applicativo.
+
+**Evidence draft** — Il documento iniziale scritto dallo specialista di contesto,
+che il sistema acquisisce e raffina in Evidence Unit. Può essere redatto e
+consegnato indipendentemente dall'installazione che userà le Evidence risultanti.
+
 **Evidence Module** — Il modulo autonomo che possiede la preparazione delle Evidence e
 la loro consultazione durante il workflow. La preparazione avviene fuori dalle singole
 sessioni; il workflow usa soltanto contenuti già pubblicati. A runtime contribuisce agli
 stage semantici esistenti, senza diventare uno stage visibile e senza modificare ledger,
 artifact o stato del workflow.
 
-**Source Evidence** — Un documento originale del workspace, conservato senza modifiche
-come riferimento umano e origine della successiva ristrutturazione.
+**Source Evidence** — Il documento o la dichiarazione che sostiene il contenuto
+corrente di una Evidence Unit. Un documento acquisito viene conservato come
+riferimento umano; una dichiarazione manuale attribuisce il contenuto alla persona
+che lo ha scritto e approvato.
+
+**Manual Evidence declaration** — Una dichiarazione esplicita dell'amministratore
+che sostiene una Evidence creata direttamente o una correzione del suo significato.
+Non implica una verifica indipendente da parte di una fonte documentale esterna.
+
+**Evidence origin** — Il documento da cui una Evidence Unit è stata inizialmente
+derivata. Può restare collegato per provenienza e confronto con gli aggiornamenti
+anche quando una dichiarazione manuale sostiene il testo corrente. La sola origine
+non dimostra il supporto semantico di una successiva correzione.
+
+**Local Evidence archive** — L'insieme delle Evidence curate custodite
+dall'installazione, distinto dalle draft originali e dai contenuti derivati per
+la ricerca. Comprende le correzioni manuali e i ritiri deliberati.
+
+**Consolidated Evidence** — Una versione delle Evidence locali controllata come
+insieme coerente e pronta per l'attivazione. I file ancora in modifica non ne
+cambiano il contenuto.
+
+**Active Evidence** — La versione consolidata disponibile alla consultazione del
+core. Un tentativo di aggiornamento fallito conserva la versione attiva precedente.
 
 **Evidence Unit** — La più piccola unità semantica coerente, revisionabile e ricercabile
-derivata da una sola Source Evidence. Possiede un identificatore stabile indipendente
-dal kind, assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono
-fuse automaticamente.
+fondata su una Source Evidence corrente, anche manuale, e con eventuale origine
+documentale distinta. Possiede un identificatore stabile indipendente dal kind,
+assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono fuse
+automaticamente.
 
 **Evidence kind** — La categoria semantica di una Evidence Unit, che ne determina i
 campi specifici e ne orienta l'uso. Ogni unità ha un solo kind primario; i tipi iniziali
@@ -151,10 +209,9 @@ avanzare fino a un retry riuscito.
 nella sessione: stage semantico, purpose, generazione interrogata e identificatori delle
 Evidence restituite. Non duplica il contenuto delle Evidence.
 
-**Curated Evidence** — Una o più Evidence Unit ristrutturate a partire da una Source
-Evidence e conservate nel repository del workspace come proposte per la revisione
-umana. Git conserva la versione precedente e rende visibile ogni modifica; una Curated
-Evidence non è ancora contenuto autorevole del runtime.
+**Curated Evidence** — Una o più Evidence Unit preparate da documenti o curate
+manualmente. La presenza nell'archivio curato non implica da sola che il contenuto
+sia già attivo per il workflow.
 
 **Published Evidence** — Le Curated Evidence valide appartenenti alla revisione attiva
 del workspace e alla generazione Evidence pubblicata. L'approvazione umana precede
@@ -176,9 +233,17 @@ una Evidence Unit. Il sistema ne verifica deterministicamente la presenza dopo l
 normalizzazione meccanica; il curatore resta responsabile di verificarne la sufficienza
 semantica.
 
-**Evidence resolution** — L'operazione esplicita con cui un curatore ritira una
-Evidence Unit oppure la ricollega a un Source Evidence esistente. Aggiorna documento e
-manifest insieme, lascia un diff Git revisionabile e non pubblica né crea commit.
+**Evidence resolution** — La decisione esplicita con cui un curatore risolve un
+problema di una Evidence Unit, correggendola, ritirandola oppure ricollegandola a
+una fonte adeguata.
+
+**Source update conflict** — Un contrasto fra una fonte aggiornata e una correzione
+manuale già approvata. La correzione resta in uso fino alla risoluzione esplicita
+del confronto da parte dell'amministratore.
+
+**Evidence source refresh** — La riacquisizione delle fonti esterne richiesta
+dall'amministratore per rilevarne le modifiche. Fra due aggiornamenti il contenuto
+già acquisito resta il riferimento per preparazione e consultazione.
 
 **Review item** — Un blocco di revisione descritto da codice stabile, messaggio umano e
 campo opzionale. Finché viene mantenuto nell'Evidence Unit, ne impedisce la
@@ -230,10 +295,18 @@ conversione degli a-capo e rimozione degli spazi esterni. Gli elementi contestua
 poi deduplicati e ordinati senza conversione delle maiuscole, mentre punteggiatura e
 spazi interni della domanda non vengono riscritti.
 
-**Additive BM25 upgrade** — L'estensione non distruttiva della collezione semantica di
-un workspace che conserva il vettore dense predefinito e aggiunge il solo vettore
-sparse `bm25`. Soltanto gli Evidence Fragment ricevono valori BM25; Schema e Memory
-mantengono invariati dati e ricerca dense.
+**Reference Vector Collection** — La collezione Qdrant ricostruibile di un workspace che
+contiene Schema, relazioni ed Evidence. Possiede il vettore dense predefinito e il vettore
+sparse `bm25`; soltanto gli Evidence Fragment ricevono valori BM25. Il preprocessing può
+sostituirla o eliminarla integralmente.
+
+**Memory Vector Collection** — La collezione Qdrant persistente di un workspace che contiene
+`memory` e `solved_question`. Non è un output del preprocessing e non viene eliminata dal
+Preprocessing Clear.
+
+**Preprocessing Clear** — L'operazione amministrativa che elimina Reference Vector Collection,
+LSH, corpus e checkpoint derivati e rende il workspace non pronto. Conserva Memory Vector
+Collection, sessioni, Catalog Metadata e database sorgente; non offre history o rollback.
 
 **Formula proposal** — Una formula individuata durante una sessione e conservata come
 artefatto della sessione. Non diventa Published Evidence finché non viene importata,
@@ -246,9 +319,10 @@ precedente o di un altro workspace.
 
 ## Configurazione dei modelli
 
-**Workspace Descriptor** — La dichiarazione versionata dell'identità e dello scope del
-Workspace Database e delle Evidence di un workspace. Non definisce modelli, selezioni di
-modello o Database Binding specifiche di un'installazione.
+**Workspace Descriptor** — La dichiarazione versionata dell'identità del workspace e dello
+scope delle sue Evidence. Non contiene identità o configurazione del Workspace Database,
+Database Binding, fatti strutturali o metadati semantici: il Metadata Catalog associa il
+workspace al relativo database.
 
 **Installation Model Catalog** — L'insieme dichiarativo, proprio di un'installazione, dei
 modelli disponibili, dei loro Model Usage e dei relativi default. È l'unica autorità per i
@@ -267,6 +341,21 @@ ridefinirne provider, endpoint o capacità.
 dell'Installation Model Catalog richiesta da uno specifico runtime. Può essere rigenerata
 integralmente dalla configurazione dell'installazione.
 
+## Distribuzione del prodotto
+
+**Customer-Hosted Installation** — Un'installazione eseguita interamente nel trust boundary
+controllato dall'organizzazione cliente, inclusi eventuali tenant cloud privati. Credenziali,
+domande, prompt, metadati e risultati non attraversano quel boundary.
+_Avoid_: on-premise deployment, self-managed deployment
+
+**Community Edition** — La distribuzione open source utilizzabile gratuitamente anche in
+produzione e capace di eseguire il workflow fondamentale completo.
+_Avoid_: free tier, trial edition
+
+**Enterprise Edition** — La distribuzione con licenza commerciale che aggiunge governance
+organizzativa, esercizio production-grade e industrializzazione alla Community Edition.
+_Avoid_: paid tier, pro edition
+
 ## Catalogo dei metadati
 
 **Workspace Database** — Il database che appartiene a un solo workspace e non può essere
@@ -287,8 +376,9 @@ client configurabile per API REST arbitrarie.
 nel catalogo autorevole. Rimane conservato per il recupero amministrativo, ma non può essere
 usato dal workflow finché non viene riassegnato a un workspace esistente.
 
-**Metadata Catalog** — Il contesto amministrativo che raccoglie e cura i metadati di un
-Workspace Database. Non definisce quali elementi partecipano al workflow SQL.
+**Metadata Catalog** — L'autorità per l'associazione fra workspace e Workspace Database, la
+relativa Database Binding, i fatti strutturali osservati e i metadati semantici curati. Ogni
+uso downstream dei metadati del database deriva da questo catalogo.
 
 **Database Profile** — L'insieme curato di scope, descrizioni e metadati semantici
 associato a un Workspace Database.
@@ -350,15 +440,19 @@ logicamente.
 Logical Relationship, con origine e stato espliciti. È l'interfaccia usata dall'amministrazione e
 dalla comprensione dello schema, non un ulteriore modello persistito.
 
-**Effective Relationship Snapshot** — La proiezione runtime immutabile delle relationship attive
-contenute nell'Effective Relationship Map. È derivata dal Metadata Catalog per una singola sessione
-e viene eliminata insieme alla relativa configurazione runtime.
+**Catalog Metadata Snapshot** — La proiezione immutabile e versionata della struttura catalogata,
+delle descrizioni pubblicabili e delle relazioni effettive attive di un Workspace Database che il
+core consuma. È derivata esclusivamente dal Metadata Catalog e non è un archivio autoritativo.
+
+**Schema Index** — La proiezione vettoriale ricostruibile dei metadati del Workspace Database nel
+Metadata Catalog. Il preprocessing la sostituisce integralmente e non è una fonte di verità.
 
 **Description** — Il testo curato e consolidato che descrive una Catalog Table o Catalog Column
-per gli usi downstream.
+per gli usi downstream. Quando presente, prevale sulla relativa Generated Description.
 
-**Generated Description** — Una proposta modificabile sottoposta a revisione umana prima di
-essere consolidata come Description. Rimane distinta dal commento osservato nel database.
+**Generated Description** — Il testo modificabile prodotto dall'AI per una Catalog Table o Catalog
+Column. È pubblicabile per gli usi downstream quando manca una Description, anche senza essere
+prima consolidato, e rimane distinto dal commento osservato nel database.
 _Avoid_: generated comment, source comment
 
 **Description Consolidation** — L'azione amministrativa esplicita che copia la Generated
@@ -404,9 +498,17 @@ Schema Synchronization.
 della Database Binding. Uno scope rimane consultabile ma è stale finché non viene sincronizzato
 con la binding corrente.
 
+**Metadata Content Revision** — La revisione monotona di tutto lo stato del Metadata Catalog che
+può modificare il comportamento del core. Ogni mutazione rilevante produce una nuova revisione
+nella stessa transazione che la rende durevole.
+
+**Preprocessing State** — Lo stato corrente `running`, `succeeded` o `failed` del preprocessing di
+un workspace, insieme all'identità dei suoi input. Il core può usare il workspace soltanto quando
+lo stato è `succeeded` e gli input coincidono ancora.
+
 **Catalog Metadata** — I campi mutabili che descrivono database, tabelle, colonne e relazioni,
 distinti dai fatti strutturali governati dalla sincronizzazione. Possono essere popolati dall'AI,
-da un'importazione o da una modifica amministrativa senza cambiare il database esterno.
+o da una modifica amministrativa senza cambiare il database esterno.
 
 **Model Completion Helper** — Il processo Python interno ed effimero che esegue una singola
 richiesta LiteLLM per conto del backend. Non è un servizio HTTP, non possiede il lifecycle della
@@ -420,6 +522,12 @@ diventa Catalog Metadata.
 **Sensitive Data Flag** — La classificazione binaria umana applicata a una Catalog Column. Può
 essere impostata liberamente dall'amministratore anche in contrasto con una valutazione automatica.
 
+**Sensitivity Reason** — La motivazione sanificata persistita insieme al Sensitive Data Flag
+quando l'amministratore salva una Sensitivity Review Draft. È Catalog Metadata della colonna, non
+history della run; viene rimossa quando il flag torna non-sensitive e può essere assente per una
+classificazione manuale priva di valutazione locale.
+_Avoid_: AI reasoning, source evidence
+
 **Local Sensitivity Assessment** — La valutazione locale, non autoritativa e priva di LLM di una
 Catalog Column, basata su metadati e contenuto sorgente, con esito `sensitive`, `non_sensitive`
 oppure `unknown`.
@@ -451,14 +559,85 @@ _Avoid_: Sensitive Data Suggestion Run, AI analysis
 
 **Sensitivity Review Draft** — La proposta transitoria che associa alle colonne selezionate una
 Local Sensitivity Assessment e le relative evidenze sanificate. Non modifica il Sensitive Data Flag
-finché l'amministratore non salva le proprie decisioni e viene scartata al reload.
+né la Sensitivity Reason finché l'amministratore non salva le proprie decisioni e viene scartata al
+reload.
 _Avoid_: automatic flag
 
 **Sensitivity Analysis Event** — Una riga testuale ordinata e sanificata che registra l'avvio,
-l'esito o l'errore di una Sensitivity Analysis Run senza conservare contenuti sorgente, output grezzi
-del detector o proposte per colonna.
+l'avanzamento per fase e batch, l'esito o l'errore di una Sensitivity Analysis Run senza conservare
+contenuti sorgente, output grezzi del detector o proposte per colonna.
 _Avoid_: Sensitive Data Suggestion Event
 
 **Introspection Capability** — Una categoria di struttura fisica che una Database Binding
 può osservare, come tabelle, colonne, relazioni, indici o enum. Una capability non disponibile
 è distinta da una capability osservata che non ha restituito elementi.
+
+## Amministrazione e integrazione
+
+**Workspace Readiness** — La preparazione di uno specifico Workspace per l'uso nel
+workflow, comprensiva della disponibilità degli artefatti derivati dai suoi metadati
+Database e dalle sue Evidence. Il preprocessing appartiene a questa preparazione;
+la configurazione e la sincronizzazione del catalogo restano responsabilità Database.
+
+**Administration Surface** — Una superficie amministrativa autonoma per configurare o curare una
+parte dell'installazione. Workspace, Evidence, Memory, Database e Pi sono superfici peer e non
+dipendono dall'esistenza di una sessione attiva.
+
+**Administration Page** — La rappresentazione a pagina intera di una Administration Surface, con
+una gerarchia condivisa per identità, stato, azioni e contenuto. Un form amministrativo appartiene
+alla pagina e non a una popup come contenitore principale.
+_Avoid_: management popup, settings modal
+
+**Administration Route** — L'identità navigabile di una Administration Surface nel browser. Deve
+essere ripristinabile con refresh e cronologia e non contiene valori transitori o segreti dei form.
+
+**Embedded Thoth Shell** — L'esperienza Thoth ospitata dentro il documento e il contesto visuale di
+un portale host. Conserva la propria gerarchia funzionale, ma deve rispettare la geometria,
+l'autenticazione e le regole responsive del portale host.
+
+**Full Thoth Shell** — L'esperienza Thoth autonoma che possiede il proprio header e il proprio
+layout di pagina. Non replica la navigazione amministrativa del portale host e non dipende dal suo
+template visuale.
+
+**Shell mode** — La scelta di installazione fra `embedded` e `full`. Determina chi possiede il
+chrome globale, i comandi di identità e le integrazioni visuali, ma non cambia il workflow o la
+persistenza delle sessioni.
+
+**Fullscreen state** — Lo stato temporaneo in cui il documento applicativo occupa il fullscreen
+del browser. È distinto da `Shell mode`: una Full Thoth Shell può essere aperta senza fullscreen;
+il passaggio è attivato da un comando esplicito e può essere annullato con la stessa azione o con
+il comando nativo del browser.
+
+**Portal Shell Adapter** — Il confine sostituibile che traduce lo stato e i comandi del chrome di
+un portale host nel modello semantico usato da Thoth. L'adapter non possiede autorizzazione,
+sessioni di workflow o contenuti del modello.
+
+**Host Shell State** — Il minimo stato visuale fornito dal portale host: locale UI, tema e stato
+fullscreen. In una Embedded Thoth Shell è la fonte autorevole per queste preferenze;
+non include identità, token o stato di autenticazione, che restano responsabilità dell'accesso.
+
+**UI locale** — La lingua delle label, dei messaggi, dei tooltip, degli stati e delle istruzioni
+non generate dal modello nell'interfaccia Thoth. È distinta dalla lingua dei contenuti di un
+workspace.
+
+**Interaction language** — La lingua in cui il modello presenta domande, spiegazioni e proposte
+al revisore durante una sessione. Viene fissata alla creazione della sessione e rimane invariata
+durante una ripresa, anche se la UI locale corrente cambia.
+
+**Administrative Page Family** — L'insieme delle cinque Administration Page che condividono shell,
+navigazione, tipografia e regole responsive, pur mantenendo contenuti e operazioni specifici:
+Workspace, Evidence, Memory, Database e Pi.
+
+## Installazione
+
+**Manual standalone installation** — Una copia di ThothII predisposta per l'uso autonomo da una
+persona che possiede il computer, con una Full Thoth Shell e servizi applicativi locali. La
+procedura non implica che DWH o provider LLM siano locali o disponibili offline.
+
+**Installation bootstrap** — L'insieme delle attività iniziali che rende disponibile una
+installazione manuale: verifica dell'host, generazione della configurazione, predisposizione
+delle credenziali protette e avvio dei servizi. Non è un installer dell'applicazione.
+
+**Platform acceptance** — La verifica che una Manual standalone installation possa essere
+predisposta e avviata su una specifica combinazione di sistema operativo, architettura e runtime,
+distinta dalla verifica funzionale del collegamento a DWH e provider LLM.

DESIGN.md

@@ -11,50 +11,50 @@ colors:
   warm-graphite: "oklch(26.78% 0.0097 355.6)"
   muted-graphite: "oklch(51.33% 0.0088 345.6)"
   quiet-border: "oklch(90.93% 0.0035 354.7)"
-  success-mint: "oklch(75.77% 0.1581 165)"
+  success-mint: "oklch(46% 0.095 160)"
   navigation-active: "oklch(92.5% 0.052 23.2)"
   navigation-active-hover: "oklch(89.5% 0.071 23.2)"
   navigation-active-foreground: "oklch(36.5% 0.11 23.2)"
   navigation-active-border: "oklch(60% 0.135 23.2)"
-  warning-amber: "oklch(85.23% 0.1386 78.9)"
-  information-blue: "oklch(70.35% 0.1128 221.3)"
+  warning-amber: "oklch(48% 0.09 70)"
+  information-neutral: "oklch(51.33% 0.0088 345.6)"
 typography:
   display:
-    fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
-    fontSize: "3rem"
+    fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
+    fontSize: "1.5rem"
     fontWeight: 600
     lineHeight: 1.03
     letterSpacing: "-0.025em"
   headline:
-    fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
-    fontSize: "1.875rem"
+    fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
+    fontSize: "1.5rem"
     fontWeight: 600
     lineHeight: 1.15
     letterSpacing: "-0.015em"
   title:
-    fontFamily: "Fraunces, Source Serif Pro, Georgia, Times New Roman, serif"
-    fontSize: "1.2rem"
+    fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
+    fontSize: "1.25rem"
     fontWeight: 600
     lineHeight: 1.25
     letterSpacing: "-0.01em"
   body:
-    fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
-    fontSize: "0.9375rem"
+    fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
+    fontSize: "1rem"
     fontWeight: 400
     lineHeight: 1.65
     letterSpacing: "normal"
   control:
-    fontFamily: "Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
+    fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
     fontSize: "0.875rem"
     fontWeight: 600
     lineHeight: 1.25
     letterSpacing: "0.005em"
   label:
-    fontFamily: "ui-monospace, SF Mono, Cascadia Code, Menlo, Consolas, monospace"
-    fontSize: "0.6875rem"
+    fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
+    fontSize: "0.75rem"
     fontWeight: 600
     lineHeight: 1.25
-    letterSpacing: "0.06em"
+    letterSpacing: "normal"
 rounded:
   xs: "4px"
   sm: "6px"
@@ -113,6 +113,16 @@ components:
 
 # Design System: ThothII
 
+## Visual review branch, September 2026
+
+The revision on `codex/ui-visual-review` is approved for implementation and Docker visual review,
+not yet for adoption on `main`. The previous look remains recoverable from the base commit and
+the preserved Docker image. Historical prototypes must remain untouched.
+
+This revision follows Impeccable's product register: one locally bundled Manrope family for the
+whole UI, five fixed size roles, red as the sole brand accent and additional color only for meaningful
+state. The primary scene remains an analyst reading data and SQL in a well-lit office.
+
 ## Overview
 
 **Creative North Star: "The Clinical Workbench"**
@@ -134,7 +144,7 @@ disciplined and tactile, never playful, sluggish, or visually unstable.
 **Key Characteristics:**
 
 - Warm, restrained surfaces with one scarce red accent.
-- Editorial headings paired with highly legible operational body text.
+- One sans-serif family, with hierarchy expressed through size, weight and spacing.
 - Dense information organized through hierarchy, rhythm, and progressive disclosure.
 - Persisted artifacts and reviewer decisions presented as the visual source of truth.
 - Fast state feedback with reduced-motion parity.
@@ -150,6 +160,15 @@ users can scan structure without adding nested containers.
 
 ## Colors
 
+The full-mode application header matches Omics Portal's `--gsd-red-primary`
+(`#CB333B`) in both themes. Its complete wordmark, including `II`, and controls
+use a near-white foreground. This header is absent in embedded mode. The sidebar
+and welcome wordmarks retain their red suffix. Context editing places workspace,
+model and Done in one desktop row, stacking on narrow containers. Session-scope
+tabs retain their selected fill and accessible keyboard state with a uniform one-pixel
+border on every side, gray when inactive and red when active. Their padding is 11px
+horizontal and 3px vertical, with a 38px minimum height and wrapping labels.
+
 The palette combines warm porcelain surfaces, warm graphite text, and an instrument red used only
 for action, focus, and important state. OKLCH values in the frontmatter are normative because the
 frontend uses OKLCH tokens directly.
@@ -178,7 +197,8 @@ frontend uses OKLCH tokens directly.
   foreground. It shares Instrument Red's hue but uses a lighter, lower-chroma fill, so location is
   visible without carrying the full weight of a primary action.
 - **Warning Amber** (`warning-amber`): waiting, attention, and in-progress states.
-- **Information Blue** (`information-blue`): informational state when red would imply action.
+- **Information**: neutral text and indicators for dates, protocols and ordinary status. The legacy
+  `--info` token resolves to muted foreground, not an additional blue accent.
 
 The dark theme keeps the same semantic mapping with neutral near-black surfaces and a slightly
 lighter red accent. Do not introduce a second visual identity for dark mode.
@@ -191,30 +211,33 @@ for their named states. Color is never the only state indicator.
 
 ## Typography
 
-**Display Font:** Fraunces, with Source Serif Pro, Georgia, and Times New Roman fallbacks  
-**Body Font:** Manrope, with native system sans-serif fallbacks  
-**Label/Mono Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks
+**UI Font:** locally bundled Manrope Variable, with Manrope and native sans-serif fallbacks.
+**Technical Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks.
 
-**Character:** Fraunces gives persisted artifacts and key headings editorial authority. Manrope
-keeps dense controls and prose calm and readable. The mono register separates machine identity,
-metadata, SQL, identifiers, and micro-labels from natural-language content.
+Manrope covers headings, labels, controls, navigation and document reading. Monospace is reserved
+for SQL, code, paths and machine identifiers, never for ordinary UI labels or status headings.
 
 ### Hierarchy
 
-- **Display** (600, `3rem`, `1.03`): authentication and exceptional page-level statements only.
-- **Headline** (600, `1.875rem`, `1.15`): major page or artifact titles.
-- **Title** (600, `1.2rem`, `1.25`): panel and document section hierarchy.
-- **Body** (400, `0.9375rem`, `1.65`): operational prose, with a target line length of 65 to 75
+- **Headline** (600, `1.5rem`, `1.3`): page or artifact titles, `--text-page`.
+- **Title** (600, `1.25rem`, `1.4`): section hierarchy, `--text-section`.
+- **Body** (400, `1rem`, `1.6`): operational prose, `--text-body`, with a target line length of 65 to 75
   characters where the surface controls width.
-- **Control** (600, `0.875rem`, `1.25`): buttons, inputs, tabs, and compact actions.
-- **Label** (600, `0.6875rem`, `0.06em` tracking): uppercase micro-labels, state metadata, and panel
-  headers. Labels use the mono family.
+- **Control** (400–600, `0.875rem`, `1.5`): buttons, inputs, tables, tabs and compact subheadings,
+  `--text-control`.
+- **Metadata** (400–600, `0.75rem`, `1.5`): secondary status, counts and timestamps, `--text-meta`.
+  Labels use sentence case and normal tracking. Ordinary operational text never falls below 12px.
 
 Typography uses fixed sizes. Responsive changes happen at structural breakpoints, not through fluid
 type scaling. Numeric data and identifiers use tabular numerals where comparison matters.
 
-**The Three Registers Rule.** Serif means authority, sans means interaction and reading, mono means
-machine identity. Do not exchange these roles for novelty.
+**Application wordmark:** ThothII is a brand mark, not a page title: use Manrope semibold at
+48px (`3rem`) in the Core welcome area and 32px (`2rem`) in the session sidebar, with the
+`II` suffix in brand red. Preserve these sizes across responsive layouts.
+
+**The One Family Rule.** The UI and document readers use sans-serif throughout. The legacy
+`--font-heading` alias resolves to `--font-sans`. Preserve technical monospace without turning it
+into a second decorative hierarchy. Do not shrink text to solve layout constraints.
 
 **The Read Once Rule.** A heading, label, and body must be distinguishable on first glance through
 size and weight. Do not repeat headings in explanatory copy.
@@ -279,27 +302,70 @@ default, hover, focus, active, disabled, loading, and error behavior where those
 - **Focus:** three-pixel Instrument Red ring with a clear border shift.
 - **Error / Disabled:** errors combine destructive color with explanatory text; disabled controls
   retain readable contrast and use 50 percent opacity.
-- **Metadata catalog model:** Database Management keeps one compact, installation-level
-  metadata-generation LLM selector in the application header. The selection persists across
-  database, table, column, and relationship views; when no usable profile is configured, the
-  disabled control explains: “No metadata-generation LLM model is configured for this installation.”
+- **Global context:** the collapsible top shelf is the sole workspace/model selector for Core and
+  Admin. Preserve independent remembered choices, installation defaults, operation locks and unsaved
+  edit guards. Never introduce a separate metadata-generation default or selector.
 
 ### Navigation
 
+- **Workspace readiness:** the Workspace navigation button carries an 8px dot to
+  the right of its label. Green means a selected workspace with confirmed ready
+  preprocessing and no query error; all other states are red. The button's
+  tooltip and accessible description retain the translated exact state. Do not
+  add a separate readiness text row or change the backend readiness gate.
+- **Session groups:** one accessible single-open accordion contains Active sessions
+  and Archive, both initially closed. Below the scope tabs, show only their
+  adjacent section headers, without a redundant Sessions heading. Selection and
+  bulk-delete controls belong inside each panel and only appear for nonempty
+  lists. Select all affects that list only, preserves the other list's selection,
+  and exposes a mixed state for partial selection. Preserve the existing archived
+  flag as the grouping rule, independent of whether a Pi process is running.
+  Opening a section closes the other; either can be collapsed, including both.
+  Empty lists show only the translated "No sessions yet." message.
+  The open section uses the rail's remaining height; its list scrolls internally
+  with a cap of `min(18rem, 35dvh)`, while its trigger remains outside that scroll
+  area. The mobile navigation dialog supplies a bounded viewport-height container.
+  Keyboard users can focus and scroll each labelled panel.
+- **Session entry:** one Session button returns to the current unfinished session,
+  including provisional creation, without resetting or reconnecting it. Otherwise
+  it prepares a new question using the normal readiness and unsaved-work guards.
 - **Style:** compact session rows use `8px` corners and restrained vertical padding.
 - **Default / Hover / Active:** porcelain at rest, Sunken Surface on hover, and a muted Navigation
   Active red with a defined border when current. Exactly one top-level navigation control is current.
-- **Administrative controls:** the admin-only Administration accordion groups Database management,
-  a structural divider, Workspace management, and Pi management in that order. Its trigger exposes
+- **Administrative controls:** the admin-only Administration accordion groups Database,
+  Memory, Evidence, a structural divider, Workspace, and Pi configuration in that order. Its trigger exposes
   expanded state and starts collapsed by default, while non-admin users do not receive the accordion
   or its navigation actions.
 - **Responsive:** collapse navigation structurally at the application breakpoint. Do not shrink
-  labels into illegibility.
+  labels into illegibility. Below 768px, Memory and Evidence management use the full content
+  width; a Navigation button opens the shared accessible dialog. Selecting another archive
+  page or pressing Escape closes it. Desktop retains the right session sidebar and its My sessions /
+  All sessions tabs. Core retains question/answer, eight phases, reviewer gates and the left log.
+  In embedded mode the portal owns the red header and left sidebar; ThothII must not duplicate them. Size to the
+  actual application container. Narrow session document panels may use the available width.
+
+### Session review and confirmations
+
+Session dialogs use the visible application area, including the portal's header
+and side rail. Artifact and schema-column review can grow to 80rem wide and the
+available height; short confirmations use up to 40rem and at least 18rem when
+space permits. Keep a 24px outer margin on desktop and 8px on small or short
+screens. Long review content scrolls internally; on very short screens the
+whole dialog can also scroll so every action remains reachable.
+
+Session forms and review gates repeat their existing primary confirmation above
+and below the content, sharing selection, validation, pending state and response
+handlers. Alternate-response inputs follow the same rule. Reserved navigation
+controls remain below the review. Stop/delete initially focus Cancel; rename
+initially focuses the name field. Administration dialogs and forms retain their
+existing layout and actions.
 
 ### Tabs
 
 - **Shape:** compact label tabs sit on a shared baseline with rounded top corners and a two-pixel
-  lower edge. Inactive labels retain a complete Quiet Border and Porcelain Card surface, so every
+  lower edge, except session-scope tabs which use a uniform one-pixel border, rounded
+  corners and a 4px gap without a shared border or negative bottom margin.
+  Inactive labels retain a Quiet Border and Porcelain Card surface, so every
   label reads as a tab before interaction; hover feedback reinforces clickability.
 - **Current:** the selected tab uses the muted Navigation Active red for its fill, text, and defined border.
   It must expose `aria-selected`, participate in a labelled `tablist`/`tabpanel`, and be the only
@@ -319,9 +385,42 @@ default, hover, focus, active, disabled, loading, and error behavior where those
 
 ### Curated Evidence Documents
 
+Memory and Evidence share the `thot-knowledge-reader` reading contract. Use locally
+bundled Manrope with normal tracking for prose and labels, and these fixed roles:
+
+- Card title: 24px, weight 600, line-height 1.3 (`thot-knowledge-title`).
+- Field/section heading, including Scope and Provenance: 20px, weight 600,
+  line-height 1.4, 8px clearance below (`thot-knowledge-heading`).
+- All narrative text, including scope, lists and provenance: 16px, weight 400,
+  line-height 1.65. Do not apply compact UI text sizes to these fields.
+- Authored Markdown subheadings inside a field: 16px, weight 600, line-height 1.5,
+  24px above/8px below. They remain subordinate to the enclosing field heading;
+  their semantic heading levels and original content are preserved.
+- Technical metadata labels/values: 14px/1.5, with weight 600 for labels.
+  Only code, paths and machine identifiers use the technical monospace family at
+  14px/1.65, identical for inline and fenced code (never compound `em` shrinkage).
+
+Separate reading sections by 24px; keep the first Markdown block flush with its
+field heading's 8px bottom gap. The same typography applies in light/dark and at
+all responsive widths. Controls and archive indexes retain their compact UI roles.
+
+Memory and Evidence detail readers use the entire available content width, without
+the ordinary 72–75ch prose cap. This is the owner's explicit reading-layout choice.
+Long unstructured paragraphs are split for display at existing sentence/semicolon
+boundaries outside inline code and links; authored Markdown structure and stored
+content are unchanged. Paragraph spacing is 1.25em. Scope and provenance share the
+available width; provenance excerpts render Markdown rather than literal markers.
+Copy actions use the two-overlapping-sheets icon, an accessible name/tooltip and
+live success/failure feedback instead of a visible Copy label.
+
+Memory has four explicitly FAKE formatting examples, one per family, in a separate
+expandable section. They reuse the real detail reader but never enter persistence,
+indexing, link search or model recall, and expose no edit/delete/save actions.
+
 Curated evidence follows a fixed reading order: title, compact type and purpose summary, scope,
-typed content, supporting excerpts, review items, then collapsed technical provenance. Machine
-metadata stays in invisible comments so GitHub Preview shows only the reviewable document.
+typed content, supporting excerpts, review items, then technical provenance. Curated v4 files
+use short, visible YAML frontmatter for identity and classification. The Markdown title and
+body are authoritative; hidden payload comments are a legacy format converted on consolidation.
 
 `applies_to` is rendered as “Ambito di applicazione” with separate bullet lists for concepts,
 tables, and columns. Enum values also use lists. Tables are forbidden for metadata, scope, or any
@@ -329,7 +428,9 @@ one-dimensional collection; reserve tables for genuinely two-dimensional dataset
 identifiers use inline code. SQL uses fenced code. Supporting excerpts use blockquotes.
 
 **The Review Surface Rule.** The visible Markdown must be readable without understanding the
-machine contract. Technical metadata belongs in progressive disclosure, not above the title.
+machine contract. In Administration, explain current and original provenance separately and
+keep file-editing templates and Git instructions in progressive disclosure. Show actual host
+paths with copy controls, never browser file links to container-only locations.
 
 ## Do's and Don'ts
 
@@ -341,7 +442,8 @@ machine contract. Technical metadata belongs in progressive disclosure, not abov
 - **Do** preserve information density with headings, rhythm, and progressive disclosure.
 - **Do** keep keyboard focus explicit and pair color with text, shape, icon, or position.
 - **Do** respect `prefers-reduced-motion` while preserving immediate non-kinetic feedback.
-- **Do** use English for interface chrome and the workspace language for persisted document content.
+- **Do** use the selected interface language (English by default) for chrome and preserve the
+  workspace language for persisted domain content. Session interaction language remains pinned.
 - **Do** render curated metadata and scope as Markdown prose or lists, never as a frontmatter table.
 - **Do** break long curated rules into paragraphs, labelled subsections, and lists at existing
   punctuation boundaries while preserving the exact canonical text for machines.

PROJECT_STATE.md

@@ -1,250 +1,101 @@
-# ThothII — Project State
+# Project state
 
-Last updated: 2026-08-31.
+Updated: 2026-09-27. This is a current snapshot, not a release diary. Stable commands
+and invariants are in [AGENTS.md](AGENTS.md); prior snapshots remain in Git.
 
-This file is the short operational snapshot. Stable commands and the architecture mental model
-live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`,
-`docs/contracts/`, `docs/adr/`, and `docs/evidence.md`. Superseded plans and reports are
-available from Git history rather than duplicated in the working tree.
+## Current contracts
 
-## Current product shape
+- React supports full/embedded rendering independently of local/OIDC/upstream auth,
+  with EN/IT UI and immutable session interaction language. See
+  [application shell](docs/architecture/application-shell.md) and
+  [localization](docs/operations/shell-and-localization.md).
+- PostgreSQL Metadata Catalog owns database identity, binding, schema, descriptions,
+  sensitivity and relationships for all core consumers. Workspace schema v4 contains
+  identity and optional Evidence only. Installation schema v2 is the authored model
+  catalog source. See [overview](docs/architecture/overview.md) and
+  [model configuration](docs/general/pi-configuration.md).
+- The harness owns workflow persistence; chat is not the durable session record.
+  Memory uses PostgreSQL authority and derived Qdrant dense/BM25 search. Editable
+  Evidence has local archive authority and manual consolidation. File save, search
+  activation and Git publication have distinct outcomes. See
+  [Memory](docs/gestione-memory.md), [Evidence](docs/contracts/curated-evidence-v4.md)
+  and [consolidated release evidence](docs/reports/knowledge-archives-release.md).
+- Reference preprocessing must not clear Memory. Use the installation-scoped
+  `tht --installation /absolute/path/thothii-installation.yaml workspace preprocess run`
+  and its [contract](docs/contracts/workspace-preprocessing-cli.md).
+- DWH sessions are read-only. SSH tunnels support database-management diagnostics
+  and metadata synchronization, not NL→SQL session creation; use direct or REST
+  transport for sessions.
 
-ThothII is a human-in-the-loop datamart builder with three independently built layers:
+## Installation and workspace boundaries
 
-```text
-frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
-```
+Fresh standalone installations follow the manual terminal procedures in
+[Italian](docs/install/standalone-manual-it.md) or
+[English](docs/install/standalone-manual-en.md), without an installer or launcher.
+The [Compose reference](docs/operations/compose-reference.md) is for maintainers,
+not another quick start. Catalog and Memory migrations are explicit.
 
-The harness owns the deterministic eight-phase NL→SQL workflow and all session persistence.
-The backend remains a process/RPC/SSE bridge for sessions and now also owns an isolated PostgreSQL
-metadata catalog for administrative database configuration. The frontend renders the review gates
-and keeps the live transcript in memory. See
-`docs/architecture/components.md` for the detailed component and data-flow map.
+Descriptors, authentication, provider credentials, certificates and runtime bindings
+stay in protected installation-local paths. Do not copy secrets into examples or
+workspace Git. Legacy runtime snapshots may use absolute paths and protected
+`harness/.env`; do not silently relocate them.
 
-## Evidence restructuring — accepted
+PSD authoring is separate at `/Users/mp/projects/tht-workspace-psd`. Its GitHub
+repository was copied to private Gitea
+[workspace_psd](https://git.tylconsulting.it/mptyl/workspace_psd), preserving both
+branches. It is a copy, not automatic synchronization. Running installations were
+not repointed to a different workspace remote.
 
-The evidence restructuring and PSD migration completed real acceptance on 2026-08-25.
+## Recorded deployments and server authority
 
-- The curated PSD revision contains 35 approved Evidence units and 60 review items.
-- The PSD authoring repository publishes all 35 units using Curated unit schema v3. Its table-free
-  presentation uses hidden canonical metadata, wrapping Markdown scope lists, list-based enum
-  values, and collapsed technical provenance. Long domain rules now have a deterministic
-  human-readable presentation while retaining their exact canonical text for vector ingestion.
-  `tht evidence migrate <workspace-root>` performs the deterministic v1/v2 upgrade and older-v3
-  presentation rewrite without model calls. The structured-rule PSD rewrite is currently local and
-  pending commit/publication.
-- The accepted snapshot is
-  `psd-clinical-675990d90eae51da6f2bd51b1ae2609f245772ef-snapshot`.
-- The active generation is `gen:f968b3bd7a553dbfef3cf47093698f2bc7f95f11`.
-- Retrieval acceptance reached 20/20 Hit@10.
-- A real session, `20301df7-cad7-403d-a4c1-9f35c9d07b66`, completed F1–F8 with five
-  receipts, three CTEs, and a final result of 78 patients.
-- The durable acceptance record is
-  `docs/testing/evidence/evidence-restructuring-psd-acceptance-2026-08-25.md`.
+Use the ordered [server handoff](docs/operations/server-codex-handoff.md) for
+coordinated ThothII/Omics upgrades. Omics integration uses GitHub
+`Dallavilla-Tiziano/omics_portal`, with no Gitea relay prerequisite. Omics uses
+embedded/upstream identity, not another ThothII OIDC login. Read
+[upstream authentication](docs/install/authentication-upstream.md) before changes.
 
-The canonical authoring, validation, publication, materialization, and preprocessing flow is
-documented in `docs/evidence.md`. The governing contracts are
-`docs/contracts/workspace-evidence-v3.md` and
-`docs/contracts/workspace-preprocessing-cli.md`.
+Last recorded application deliveries (not a fresh runtime attestation):
 
-## Workspace preprocessing and configuration
+- [Coordinated ThothII/Omics release](docs/reports/2026-09-26-server-release-execution.md):
+  core/frontend `497ab840-preflight`, Omics proxy fix `928f7e9f` with existing web
+  image retained. Automated acceptance passed; on September 27 the operator confirmed
+  browser access, UI controls and session start/stop/resume. Functional browser
+  acceptance passed; remaining extended checks are handed off in the
+  [server acceptance follow-up](docs/reports/2026-09-27-server-acceptance-handoff.md).
+- [Session dialogs](docs/reports/2026-09-14-session-dialogs-release.md):
+  `b1723c34-session-dialogs-20260914`, frontend-only.
+- [Session layout/Memory fix](docs/reports/2026-09-14-session-layout-memory-fix.md):
+  `49333a2d-session-memory-fix`, core/frontend.
 
-The native host CLI `tht` is the operator surface. Workspace preprocessing runs through:
+Keep those reports and rollback instructions while operator gates remain open.
+A later deployment does not prove every earlier acceptance item passed.
 
-```sh
-tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
-tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
-```
+## Remaining acceptance and design gates
 
-These commands use the profile-gated `workspace-maintenance` service. The former standalone
-preprocessing Compose fixtures are retired.
+- Fresh-machine Mac, Windows/WSL2 and Linux installation acceptance, including real
+  DWH/model endpoints, remains a separate operator exercise.
+- Real IdP/portal login, logout, embedded interaction and PSD semantic acceptance
+  follow the [manual matrix](docs/testing/authentication-manual-acceptance.md) and
+  delivery reports; synthetic tests do not close them.
+- [Security hardening](docs/plans/2026-09-08-security-hardening-prd.md) is a draft.
+  Revalidate SEC01–12 and obtain design approval before implementation or real
+  server/IdP/DWH mutation.
+- Optional NER remains opt-in; labeled Italian quality, benchmark and licensing
+  acceptance are not implied by document cleanup.
+- Semantic aliases, value descriptions, synonyms/concepts, dialect and multi-schema
+  extensions remain explicit design work. Current sensitivity delivery follows the
+  Catalog contract; additional policies require their own acceptance.
+- Legacy database UI fallback (`?db-ui=legacy`, dev/staging) and prototype removal
+  remain subject to owner acceptance.
 
-Workspace descriptors use schema v3. For PSD, workspace content and runtime roots point to the
-separate uncommitted repository `/Users/mp/projects/tht-workspace-psd`. Secrets remain outside
-Git and are supplied only through installation-local protected files.
+## Documentation maintenance
 
-## Database management
+MkDocs publishes only 20 product/operator pages and five approved assets.
+Architecture, contracts, ADRs, plans, research, tests and release evidence are
+excluded from HTML and search. The repository itself is public: editorial exclusion
+is not confidentiality.
 
-The database, table, and authoritative physical-schema catalog slices are implemented. Database
-Management now opens the Fleet Ledger presentation by default inside `AppShell`, lists every YAML
-workspace, creates at most one PostgreSQL database configuration per workspace, edits direct
-PostgreSQL, REST API, or SSH-tunnel installation bindings, replaces write-only encrypted secrets,
-and tests supported connector bindings. The surface keeps one responsive AG Grid visible at a time:
-databases lead to tables, tables lead to columns, and relationships are a sibling database view.
-Parent navigation remains explicit through the breadcrumb and emphasized back control.
-
-Selection-scoped operations use one action selector plus an explicit **Run** control; ineligible
-actions remain visible with their disabled reason, while row-scoped actions stay in the pinned final
-column. The KPI strip reads installation-wide or selected-database aggregates from
-`GET /catalog/metrics`. Database configuration, metadata editors, synchronization history,
-description history, and sensitive-field review/history use the production APIs in right-side
-drawers rather than prototype fixtures; closing a history drawer does not stop its background run.
-
-Physical membership, source
-comments, column types/default/nullability/PK positions, and constraint-level ordered FK pairs are
-projections of the external schema. They cannot be created, renamed, or structurally edited by
-hand, but administrators can explicitly clear catalog tables, columns, or relationships without
-touching the source database, binding, configuration, or secrets. Table deletion cascades through
-columns and relationships; table-scoped relationship cleanup includes incoming and outgoing
-relationships. Curated and generated descriptions are editable; generated descriptions start null
-and Database Management can generate or consolidate them for selected tables, selected columns,
-all targets, or only targets whose Generated Description is missing.
-
-Relationship Management is now reachable directly from each configured Fleet database. One
-Relationship Map shows read-only Physical Relationships together with Generated and Manual Logical
-Relationships, with Active, Excluded, and All filters. Administrators can add a single-column
-relationship, run deterministic name/PK/type inference, exclude or restore a logical relationship,
-or delete it permanently. Exclusion retains a tombstone that a rebuild cannot reactivate; permanent
-deletion allows a later rebuild to infer the same endpoints again. Inference uses no LLM, embedding,
-or source values. It supports normalized table-qualified names, unique non-generic PK names,
-composite-PK source columns, and the `*time_key -> dim_time.<single PK>` warehouse convention while
-ignoring bare generic names. Explicit table/column metadata cleanup remains a destructive boundary: it removes
-the attached logical relationships and exclusions and requires a full schema synchronization before
-inference or runtime publication can continue.
-
-The previous Database Management renderer remains a temporary comparison fallback for development
-and staging only: `?db-ui=legacy` is honored in Vite development or when
-`VITE_DB_MANAGEMENT_LEGACY=true`; it is not a production presentation. The standalone Fleet Ledger
-prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
-after which both migration aids can be removed.
-
-Schema refresh is one durable asynchronous engine with database-table, selected-table-column,
-relationship, and full-database actions. Database-level menus expose only the table, relationship,
-and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections
-also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
-one-active-operation-per-database exclusion with synchronization. Runs have leases and
-restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
-apply, retained history, and a live SSE log with polling fallback. Null metadata renders blank
-rather than as a placeholder.
-
-Direct PostgreSQL and strict known-host-verified OpenSSH use `pg_catalog`. REST bindings use the
-typed full-snapshot `POST /rpc/schema_snapshot` contract when available. Servers such as the
-current PSD endpoint that exposes only `POST /rpc/run_query` use one catalog-owned read-only query
-to return the exact same strict v1 snapshot in a single round trip. Both paths remain fail-closed:
-an absent capability, query error, partial result, or invalid snapshot applies no catalog changes.
-SSH is not yet enabled for NL→SQL session runtime.
-
-The catalog runs in the internal `catalog-db` PostgreSQL service. Kysely migrations are an explicit
-one-shot `catalog-migrate` operation; `scripts/run-stack.sh` runs it before local startup. Runtime
-sessions now consume the Catalog's active effective relationship map through an immutable JSON
-snapshot tied to the runtime-config lease. The harness uses that snapshot as its exclusive
-relationship source while retaining Git-pinned annotations for descriptive metadata; legacy
-runtimes without a snapshot keep the previous merge behavior. The accepted design is recorded in
-`docs/plans/2026-08-26-metadata-catalog-from-thothai.md`, the snapshot contract under
-`docs/contracts/`, and ADRs 0001–0012.
-
-Semantic aliases, value descriptions, synonyms, and concepts remain deferred to their dedicated
-slices.
-
-AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to
-`false`, including for newly synchronized columns. An administrator may request an AI proposal based
-only on structural metadata for one selected database, selected tables, or selected columns. The
-backend divides large scopes into deterministic model requests of at most ten columns, also bounded
-by helper message size, and combines their results, but the proposal remains an unsaved draft until
-the human reviews and saves it.
-Each started suggestion attempt records a separate Sensitive Data Suggestion Run with aggregate
-counters and safe ordered events. This operational history never stores per-column proposals,
-prompts, raw model output, or provider diagnostics; reloading still discards an unsaved review
-draft.
-For unprotected columns, up to five source rows and five representative non-null values may be sent
-transiently to the configured model provider. Protected columns are omitted from source reads and
-replaced in the prompt by deterministic plausible values derived only from their metadata. Existing
-descriptions are not regenerated when a flag changes.
-
-The accepted AI-description design is recorded in
-`docs/plans/2026-08-28-ai-catalog-description-generation.md`, with the formal specification in the
-adjacent `-spec.md` document and Gitea issue #4. Gitea issues #5–#11 deliver the implementation.
-The runtime deliberately keeps ThothAI's simple operating model: one installation-wide sequential
-run owned by the backend, one short-lived Python/LiteLLM completion helper per request, and
-persistence limited to the run, its safe ordered text events, and each Generated Description as
-soon as it succeeds. The helper performs at most one provider retry and never falls back to another
-model. Stop terminates the current helper and retains prior results; three consecutive exhausted
-technical batches fail the run. Startup marks stale queued/running work interrupted, and Unlock is
-available only when no local start, worker, or helper is live. Runs remain inspectable through a
-live SSE log with ordered polling fallback; there is no automatic resume or user-facing generation
-CLI. ADRs 0009–0010 record the runtime and source-sampling decisions.
-
-Metadata-generation setup accepts the protected `DEEPSEEK_API_KEY` and `ZAI_API_KEY` references.
-It also accepts a model with no secret reference only when its OpenAI-compatible endpoint is
-explicit; this covers the VPN-only AritmoLab Qwen 3.6 server without creating a fake operator
-credential. The Python client supplies only its fixed non-secret compatibility placeholder.
-The AritmoLab entry also sets `disableThinking: true`, mapped to the endpoint's chat-template flag,
-because its default reasoning prose would violate the worker's exact JSON response contract.
-
-Logical relationship integration with core schema-linking is complete: session creation and resume
-materialize the active physical/generated/manual map, retrieval-pack generation and Pi receive the
-same runtime config, and snapshot validation fails closed on a declared missing, invalid, or orphaned
-endpoint. Broader publication of other Catalog metadata to schema-linking remains a separate future
-slice.
-
-**Deferred follow-up — Sensitive Data Policy in schema-linking.** The policy is first delivered
-and tested in catalog description generation. Its enforcement for core schema-linking remains
-out of scope until the current tickets are closed and the owner has completed the acceptance test.
-At that gate, resume the design: `tht` must receive a read-only projection of the current Sensitive
-Data Flags and exclude values from columns marked sensitive from every LSH result before it is
-given to Pi. Do not start this integration before the owner gives final approval after that test.
-
-## Active deployment work and manual gates
-
-### PSD server deployment program
-
-The approved design and executable entry point are:
-
-- `docs/plans/2026-08-20-psd-server-deployment-program-design.md`
-- `docs/plans/2026-08-20-psd-server-deployment-program.md`
-- `docs/plans/2026-08-20-psd-server-survey.md`
-- `docs/plans/2026-08-20-psd-server-project-a-standalone.md`
-- `docs/plans/2026-08-20-psd-server-project-b-authentik.md`
-
-Last recorded state:
-
-- survey: `SURVEY_NO_GO`;
-- Project A: `BLOCKED_BY_SURVEY_AND_MUTATION_GATE`;
-- Project B: `BLOCKED_BY_PROJECT_A_AND_PRE_B_GATE`.
-
-The deployment is a clean replacement: legacy sessions, indexes, and application configuration
-are not migration inputs. The existing stack remains intact until its documented mutation and
-rollback gates are explicitly approved. Shared Omics/LocalLLM networks, ETL Evidence, DWH,
-`dwh-auth`, Supabase, Authentik, Superset, and Aritmolab are outside cleanup scope.
-
-Human acceptance guides and sanitized report templates live under `docs/testing/` and
-`docs/testing/evidence/`. The remediation checklist is
-`docs/operations/psd-server-survey-remediation-checklist.md`.
-
-### Authentication
-
-The local/OIDC authentication remediation passed its automated review on 2026-08-18. Release and
-PSD mutation gates remain governed by:
-
-- `docs/architecture/authentication.md`;
-- `docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md`;
-- `docs/operations/psd-dwh-auth-rollout.md`;
-- `docs/testing/authentication-manual-acceptance.md`.
-
-Do not infer authorization for server, Nginx, Authentik, database, credential, or cutover changes
-from an automated PASS.
-
-## Verification status
-
-- The P1.1 workspace-directory registry and P2–P6 preprocessing workstreams are implemented and
-  have automated coverage.
-- Evidence restructuring has a real PSD acceptance PASS as recorded above.
-- AI Description Generation has automated coverage across installation setup, model selection,
-  generation/consolidation scopes, bounded sampling, cancellation/recovery, history, SSE/polling,
-  and the LiteLLM helper boundary.
-- L2 tests requiring real providers or remote databases remain opt-in.
-- Server deployment, release, and owner-operated acceptance steps remain pending wherever the
-  referenced runbooks require explicit approval.
-
-Run the layer-specific checks documented in `AGENTS.md`. For release-sensitive changes, also run
-the repository contract scripts in `scripts/` and build the MkDocs site.
-
-## Operational invariants
-
-- `tht`'s `-c`/`--config` option follows the subcommand; it is not a global option.
-- `--json` commands write pristine JSON to stdout.
-- Persisted phase documents and the decision ledger are the source of session truth; chat is not.
-- UI chrome is English; workspace document content retains the workspace language.
-- The backend refuses resume for finalized or archived sessions.
-- A resume must send `/riprendi-sessione <id>`; a new session must send `/nuova-domanda`.
-- DWH access is read-only.
+The [cleanup record](docs/maintenance/2026-09-15-documentation-cleanup.md) records
+retired sources and retained gates. Main contains source; Actions generates the
+`pages` branch. The live site requires the separate explicit deployment described
+in [public manual publication](docs/operations/public-docs-publication.md).

README.md

@@ -4,72 +4,51 @@ ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fasti
 core. The portable deployment runs two application services plus the installation-local metadata
 catalog; DWH and LLM services remain external. Semantic services are bundled in Compose.
 
-Authentication is configured through the single host CLI tht: see the [local authentication guide](docs/install/authentication-local.md),
-[generic OIDC guide](docs/install/authentication-oidc.md), and [manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
+The same frontend supports **full** (its own header) and **embedded** (inside a
+portal). This choice is independent of authentication: the Mac uses full/local,
+Omics uses embedded/upstream with its existing login, and a standalone server
+can use full/OIDC. See [rendering architecture](docs/architecture/application-shell.md)
+and [configuration, Omics delivery and deploy](docs/operations/shell-and-localization.md).
 
-## Docker Compose: local startup
+For the current server upgrade with Omics Portal, follow the ordered
+[Codex server handoff](docs/operations/server-codex-handoff.md), including source
+integration, embedded/upstream configuration, coordinated rollout and rollback.
 
-Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`,
-`catalog-db`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external,
-configurable endpoints—even when they are co-located with ThothII.
+Local/OIDC authentication is configured through the host CLI `tht`; portal
+authentication is established by the trusted server proxy. See the
+[local guide](docs/install/authentication-local.md),
+[OIDC guide](docs/install/authentication-oidc.md),
+[upstream integration](docs/install/authentication-upstream.md), and
+[manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
 
-From a fresh clone, run these commands from the repository root:
+For the clone-based manual standalone installation test on macOS, Windows, and Linux, use the
+[Italian procedure](docs/install/standalone-manual-it.md) or the
+[English procedure](docs/install/standalone-manual-en.md).
 
-```sh
-cp deploy/env/local.env.example deploy/env/local.env
-# Copy docs/install/examples/thothii-installation.local.yaml to a protected operator path,
-# replace its placeholders, chmod it 600, and set that exact THT_INSTALLATION_CONFIG_SOURCE.
-# Edit deploy/env/local.env, including PI_AUTH_FILE, THT_SECRETS_FILE, and external endpoints.
-./scripts/run-stack.sh
-```
+The [public manual](https://git.tylconsulting.it/thothii-docs/) covers the product,
+installation, use and administration. Developer architecture, contracts, ADRs, tests,
+plans and release records remain in this repository but are excluded from MkDocs
+pages and search. This is an editorial boundary, not an access restriction on the
+public repository. See the [documentation cleanup review](docs/maintenance/2026-09-15-documentation-cleanup.md)
+for the executed consolidation and the inventory of historical sources retained in Git.
 
-The launcher builds the core, starts `catalog-db`, runs the explicit one-shot Kysely migrations,
-then runs the base+local stack in the foreground. Migrations never run implicitly in backend
-startup. The core image contains its Pi runtime; no host `pi` executable is used. For a server
-installation, build the image, start the catalog, and run the same migration service before the
-application rollout:
+## Docker Compose and installation
 
-```sh
-cp deploy/env/server.env.example deploy/env/server.env
-# Prepare a mode-600 thothii-installation.yaml from the server example and set its exact
-# path as THT_INSTALLATION_CONFIG_SOURCE. Edit all remaining storage/secret/endpoint paths.
-sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001
-docker compose --env-file deploy/env/server.env \
-  -f compose.yaml -f deploy/compose.server.yaml \
-  -f deploy/compose.session-server.yaml.example build core
-docker compose --env-file deploy/env/server.env \
-  -f compose.yaml -f deploy/compose.server.yaml \
-  -f deploy/compose.session-server.yaml.example up -d catalog-db
-docker compose --env-file deploy/env/server.env \
-  -f compose.yaml -f deploy/compose.server.yaml \
-  -f deploy/compose.session-server.yaml.example run --rm catalog-migrate
-docker compose --env-file deploy/env/server.env \
-  -f compose.yaml -f deploy/compose.server.yaml \
-  -f deploy/compose.session-server.yaml.example up --build -d
-```
+For a fresh installation, follow the complete manual procedure in
+[Italian](docs/install/standalone-manual-it.md) or
+[English](docs/install/standalone-manual-en.md). Configure protected files first;
+then run the documented build, explicit migrations and startup commands with the
+same installation descriptor and Compose project. There is no installer or launcher.
 
-The initializer is required for an empty or restored server Pi-state bind. It atomically creates
-the three regular targets hidden below the writable parent bind; protected Pi auth and tracked
-model/settings sources remain separate read-only mounts. See the server manual before substituting
-a root other than `/srv/thothii/pi-state`.
+The mandatory stack includes frontend, core, PostgreSQL catalog, Qdrant, Ollama
+and the embedding initializer. DWH and LLM endpoints remain external dependencies.
+Pi is included in the core image. Credentials and certificates belong in protected
+installation-local files, never in the workspace repository.
 
-Workspace descriptors come from the Git remote configured by `THT_WORKSPACE_GIT_REMOTE`; their
-runtime endpoint and secret bindings remain installation-local. Open
-<http://127.0.0.1:8080> (set `THOTH_HTTP_PORT` in `deploy/env/local.env` to choose another
-loopback port).
-
-Credentials and certificates are local protected files. Do not put them in environment examples,
-workspace YAML, URLs, or Compose interpolation values.
-
-Application state is split across the named `settings`, `pi-state`, `workspace-registry`,
-`sessions`, `qdrant-data`, and `embedding-models` volumes. `docker compose down` keeps them.
-`qdrant-data` is a derived but persistent index store; `embedding-models` is an Ollama model
-cache for `qwen3-embedding:0.6b` with fixed `1024`-dimension embeddings. Only an explicit destructive command such as `docker compose
-down --volumes` removes them.
-
-The frontend depends on the core health check and proxies `/health` and `/api/*` to it. The
-application health endpoint intentionally checks process readiness only; external dependency
-diagnostics are exposed by `tht doctor` and do not prevent the UI from starting.
+For developer topology, overlays and lifecycle details, see the internal
+[Compose reference](docs/operations/compose-reference.md). Ordinary stop/down keeps
+persistent data; removing volumes is destructive and is not an upgrade step.
+Process health is distinct from external dependency checks performed by doctor.
 
 ## Git-backed workspace repository
 
@@ -106,15 +85,20 @@ a remote user's partial list. The isolated deployment exercise is
 `./scripts/verify-workspace-install-docs.sh --profile local` or `--profile server`.
 
 <!-- workspace-descriptor-contract:start -->
-Schema v3 is the only accepted workspace descriptor. Schema v1 and v2 workspace descriptors are
-rejected before activation. Candidate snapshot validation therefore makes activation or a pull fail
-atomically while the prior valid snapshot remains active. There is no in-product migrator or
-automatic conversion. A repository must already contain reviewed v3 descriptors. One workspace
-owns one Qdrant collection;
-schema, Evidence, and Memory records share that collection and stay separated by indexed payload
-`kind`.
+Schema v4 is the only accepted workspace descriptor. It contains workspace identity and optional
+Evidence configuration only; PostgreSQL Metadata Catalog owns every database fact and binding.
+Schema v1, v2, and v3 descriptors are rejected before activation. Candidate snapshot validation
+therefore makes activation or a pull fail atomically while the prior valid snapshot remains active.
+Each workspace owns separate Qdrant `reference` and `memory` collections: Schema, relationships, and
+Evidence are replaceable reference data; Memory and solved questions have a persistent lifecycle.
 <!-- workspace-descriptor-contract:end -->
 
+<!-- non-workspace-migration:start -->
+Create a clean v4 descriptor containing only `workspace` and optional `evidence`. Do not copy the
+legacy database, diagnostics, `llm_policy`, or `semantic_index` blocks; configure the database in
+Database Management.
+<!-- non-workspace-migration:end -->
+
 For NL→SQL runtime sessions, connector `ssh_tunnel` bindings remain diagnostic-only: their bounded
 probe cleans up the loopback forward and returns `workspace_not_activatable`; session creation is
 rejected before persistence. Database management is a separate boundary and supports a strict
@@ -176,10 +160,10 @@ secret files, upstream-auth checks, and a fail-closed `503` assertion for its de
 unavailable disposable session endpoint. No real provider, database credential, or repository
 secret is required.
 
-For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular
-`agent/auth.json`, `agent/models.json`, and `agent/settings.json` mount targets atomically before
-Compose. The server smoke starts from an empty Pi-state root and applies this same preflight; the
-real protected/tracked sources remain separate read-only mounts. Deterministic fixture tests render
+For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular Pi agent
+mount targets atomically before Compose. The auth target receives the protected credential bind;
+the model and settings targets receive generated read-only projections. The server smoke starts
+from an empty Pi-state root and applies this same preflight. Deterministic fixture tests render
 both profiles, verify that bindings stay on `core`, check mount readability, and run the production
 workspace resolver. Wrong-service, wrong-value, and broken-secret-mount mutations must fail.
 
@@ -189,7 +173,7 @@ an independent 32-minute outer timeout and does not retry a failed command.
 Current release status (2026-08-05): clean-root render/setup and the production runtime-binding
 resolver contracts are green. The server fixture supplies all four private trusted claims,
 including exact non-admin value `0`, and a focused test proves nginx normalization produces the
-accepted non-admin backend principal. Canonical schema-v3 registry descriptors now pass through
+accepted non-admin backend principal. Canonical schema-v4 registry descriptors now pass through
 one backend-owned, secret-safe runtime handoff for inventory and session execution; canonical
 identity and durable session/artifact/index roots are retained. The fresh update-only smoke passed
 bad-candidate mutation, automatic `rolled_back` compensation, exact prior-image restoration,
@@ -220,15 +204,23 @@ job remains deterministic and does not claim Docker startup.
 
 ## Workspace preprocessing and S3 Evidence
 
-Run preprocessing through the native host CLI and the installation descriptor:
+For an interactive run, select the workspace, expand **Administration** in the right sidebar, and
+use its **Preprocessing** control. The control explains any unmet prerequisite and exposes only the
+latest safe failure diagnostic. For unattended operation, use the native host CLI and installation
+descriptor:
 
 ```sh
-tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
-tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
+tht --installation /absolute/path/thothii-installation.yaml \
+  workspace preprocess run --workspace <workspace-id>
+
+tht --installation /absolute/path/thothii-installation.yaml \
+  workspace preprocess clear --workspace <workspace-id>
 ```
 
-The CLI starts the profile-gated `workspace-maintenance` service and enforces the workspace,
-secret, Qdrant, and embedding contracts. See [Evidence](docs/evidence.md) and the
+The one-shot command starts the profile-gated `workspace-maintenance` service, reads database
+metadata from PostgreSQL, and rebuilds LSH plus schema/Evidence vectors. The clear command removes
+those derived artifacts while preserving the separate Memory collection. The core remains unavailable
+until preprocessing completes. See [Evidence](docs/evidence.md) and the
 [workspace preprocessing CLI contract](docs/contracts/workspace-preprocessing-cli.md).
 
 S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance.
@@ -269,7 +261,7 @@ Compose project name by passing `--confirm-project`:
 The restore script stops `qdrant`, validates the exact labeled target, stages the current volume
 contents for rollback, extracts the requested archive into the volume, and then returns the
 service to its prior running state. It restores semantic storage only. Before reopening write
-traffic, the workspace registry must already be at a reviewed v3 descriptor revision compatible
+traffic, the workspace registry must already be at a reviewed v4 descriptor revision compatible
 with the restored collection; then run backend health checks and a known retrieval query. The
 helper does not restore descriptors, rename collections, or reconcile an incompatible collection
 contract.
@@ -295,14 +287,12 @@ Copy `deploy/secrets/thothii.secrets.example` to a protected host file, include
 keys, and set its absolute path as `THT_SECRETS_FILE` in the operator env. Keep Pi's native
 provider auth in the separate protected file named by `PI_AUTH_FILE`.
 
-Description Generation is configured independently in the protected installation descriptor under
-`metadataGeneration`. Set `THT_INSTALLATION_CONFIG_SOURCE` to that exact host file; Compose mounts
-it read-only into `core` and supplies the fixed runtime `THT_INSTALLATION_CONFIG_FILE` path. Each
-keyed model stores only an audited `apiKeyEnv` reference. The referenced value stays in the secret
-bundle; a model may omit `apiKeyEnv` only when it declares an explicit endpoint that accepts
-unauthenticated requests. The browser receives only model IDs, labels, and the configured default.
-Configuration changes take effect after restart and do not use Pi settings or workspace
-`llm_policy`.
+Interactive sessions, Description Generation, and embedding share the protected installation
+descriptor's `modelCatalog`. Set `THT_INSTALLATION_CONFIG_SOURCE` to that exact host file; `tht`
+validates it and generates the runtime catalog, Pi adapters, and Compose override before startup.
+Each authenticated provider stores only an audited `apiKeyEnv` reference; the referenced value stays
+in the secret bundle. A provider may use `authentication.mode: none` only with an explicit keyless
+endpoint. The browser receives only eligible model IDs, labels, and the catalog default.
 
 Before enabling Description Generation, approve the selected model provider for bounded source-data
 disclosure. Every catalog column has a **Sensitive** flag that defaults to `false`. Administrators can
@@ -333,22 +323,11 @@ the host/secret-manager materialization and add a reviewed Compose override that
 does not create that mount. The frontend remains on loopback; the authenticated host proxy is the
 only public listener.
 
-Set the selected model provider in application settings (or `PI_PROVIDER`). For each Pi spawn the
-backend validates and reads `THT_MODEL_API_KEY` from the bundle, then exposes its value only as the provider's
-recognized child variable (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or
-`ZAI_API_KEY`). Neither the generic file path nor deprecated `PI_PROVIDER_API_KEY` is inherited by
-Pi. Local providers such as Ollama require no model key.
-
-`THT_MODEL_API_KEY` supports Pi providers whose authentication is exactly one key:
-`ant-ling`, `anthropic`, `cerebras`, `deepseek`, `fireworks`, `github-copilot`, `google`
-(including the `gemini` alias), `google-vertex` when using its API-key mode, `groq`,
-`huggingface`, `kimi-coding`, `minimax`, `minimax-cn`, `mistral`, `moonshotai`,
-`moonshotai-cn`, `nvidia`, `openai`, `opencode`, `opencode-go`, `openrouter`, `together`,
-`vercel-ai-gateway`, `xai`, the four `xiaomi*` providers, `zai`, and `zai-coding-cn`.
-Compound providers are deliberately unsupported: `amazon-bedrock`, `azure-openai-responses`,
-`cloudflare-workers-ai`, and `cloudflare-ai-gateway` require multiple credential/configuration
-values. Selecting one fails before Pi starts; ambient AWS, Azure, and Cloudflare credentials are
-still scrubbed. Supporting them requires a future dedicated provider-specific configuration.
+For each Pi spawn, the backend resolves the selected canonical provider/model in the runtime catalog,
+reads exactly that provider's declared `apiKeyEnv` value from the bundle, and exposes only that key
+to the child. Ambient provider credentials and secret-bundle paths are scrubbed. Providers needing a
+compound credential bundle remain unsupported until the catalog gains an explicit generic contract
+for them.
 
 ## User-owned session server cutover
 
@@ -357,6 +336,8 @@ The server profile stores sessions and per-user preferences directly in PostgreS
 dual write. Use [`deploy/compose.session-server.yaml.example`](deploy/compose.session-server.yaml.example)
 with the canonical base+server files and set `THT_SERVER_WORKSPACE_CONFIG` to an absolute,
 protected copy of [`deploy/workspaces/server-sessions.yaml.example`](deploy/workspaces/server-sessions.yaml.example).
+That file is an installation runtime template, not an authored workspace descriptor; database
+bindings are injected from the PostgreSQL Metadata Catalog for each runtime lease.
 
 The runtime login needs membership in the no-login database role `thoth_sessions_runtime` only.
 The distinct, one-shot migrator login needs migration authority and uses

backend/package-lock.json

@@ -12,14 +12,17 @@
         "@types/pg": "^8.20.3",
         "fastify": "^5.0.0",
         "kysely": "^0.29.5",
+        "libphonenumber-js": "1.13.12",
         "openid-client": "6.8.5",
         "pg": "^8.22.0",
+        "validator": "13.15.35",
         "yaml": "^2.9.0",
         "zod": "^4.4.3"
       },
       "devDependencies": {
         "@testcontainers/postgresql": "^12.1.0",
         "@types/node": "24.13.3",
+        "@types/validator": "13.15.10",
         "tsx": "^4.19.0",
         "typescript": "^5.6.0",
         "vitest": "^2.1.0"
@@ -1329,6 +1332,13 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/validator": {
+      "version": "13.15.10",
+      "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.10.tgz",
+      "integrity": "sha512-T8L6i7wCuyoK8A/ZeLYt1+q0ty3Zb9+qbSSvrIVitzT3YjZqkTZ40IbRsPanlB4h1QB3JVL1SYCdR6ngtFYcuA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@vitest/expect": {
       "version": "2.1.9",
       "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz",
@@ -2458,9 +2468,9 @@
       }
     },
     "node_modules/fast-uri": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
-      "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
+      "version": "3.1.7",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz",
+      "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==",
       "funding": [
         {
           "type": "github",
@@ -2474,9 +2484,9 @@
       "license": "BSD-3-Clause"
     },
     "node_modules/fastify": {
-      "version": "5.8.5",
-      "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.8.5.tgz",
-      "integrity": "sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==",
+      "version": "5.12.3",
+      "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.12.3.tgz",
+      "integrity": "sha512-reZ8wce5VNCcufIt9AVtzZa3L4u1j8esikn7OEgHWLVpRpL5R7Y2+Xzj70OUkv5zDfzUAxXZT6cu4Rt0zr3EKA==",
       "funding": [
         {
           "type": "github",
@@ -2495,11 +2505,11 @@
         "@fastify/proxy-addr": "^5.0.0",
         "abstract-logging": "^2.0.1",
         "avvio": "^9.0.0",
-        "fast-json-stringify": "^6.0.0",
-        "find-my-way": "^9.0.0",
+        "fast-json-stringify": "^7.0.0",
+        "find-my-way": "^9.6.0",
         "light-my-request": "^6.0.0",
         "pino": "^9.14.0 || ^10.1.0",
-        "process-warning": "^5.0.0",
+        "process-warning": "^5.1.0",
         "rfdc": "^1.3.1",
         "secure-json-parse": "^4.0.0",
         "semver": "^7.6.0",
@@ -2522,6 +2532,46 @@
       ],
       "license": "MIT"
     },
+    "node_modules/fastify/node_modules/fast-json-stringify": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-7.0.1.tgz",
+      "integrity": "sha512-eRSayARSbbwlBjpP4vnTTIRD5QPcIrmihPxDeN1DtKnHPg66UuJLx+8hlK1kaFdjvzyQ/dzALoi4vwAQ+T+iZA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fastify"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fastify"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "@fastify/merge-json-schemas": "^0.2.0",
+        "ajv": "^8.12.0",
+        "ajv-formats": "^3.0.1",
+        "fast-uri": "^4.0.0",
+        "json-schema-ref-resolver": "^3.0.0",
+        "rfdc": "^1.2.0"
+      }
+    },
+    "node_modules/fastify/node_modules/fast-uri": {
+      "version": "4.1.4",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.4.tgz",
+      "integrity": "sha512-dODXrIxlS9JSdgAnhIUKOosKV1oMtU2VtVw87QRaHzyl5jxO290Ii5tEZfCfzfWNHi3jKWwBSdQj0qIyshdZdQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fastify"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fastify"
+        }
+      ],
+      "license": "BSD-3-Clause"
+    },
     "node_modules/fastq": {
       "version": "1.20.1",
       "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
@@ -2824,6 +2874,12 @@
         "safe-buffer": "~5.1.0"
       }
     },
+    "node_modules/libphonenumber-js": {
+      "version": "1.13.12",
+      "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.13.12.tgz",
+      "integrity": "sha512-uLVeV1c9OTk6qkdqnj+mpMD+ZdnZ0szVyWu58HwMmpwkHA1gCEkyjd3veZQXDnuw9KEwSRjcc9B1pS9XKIN1fA==",
+      "license": "MIT"
+    },
     "node_modules/light-my-request": {
       "version": "6.6.0",
       "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz",
@@ -2971,9 +3027,9 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "3.3.15",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz",
-      "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==",
+      "version": "3.3.18",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz",
+      "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==",
       "dev": true,
       "funding": [
         {
@@ -3225,9 +3281,9 @@
       "license": "MIT"
     },
     "node_modules/postcss": {
-      "version": "8.5.15",
-      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
-      "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
+      "version": "8.5.28",
+      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz",
+      "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==",
       "dev": true,
       "funding": [
         {
@@ -3245,7 +3301,7 @@
       ],
       "license": "MIT",
       "dependencies": {
-        "nanoid": "^3.3.12",
+        "nanoid": "^3.3.18",
         "picocolors": "^1.1.1",
         "source-map-js": "^1.2.1"
       },
@@ -3310,9 +3366,9 @@
       "license": "MIT"
     },
     "node_modules/process-warning": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz",
-      "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==",
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz",
+      "integrity": "sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==",
       "funding": [
         {
           "type": "github",
@@ -4121,6 +4177,15 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/validator": {
+      "version": "13.15.35",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.35.tgz",
+      "integrity": "sha512-TQ5pAGhd5whStmqWvYF4OjQROlmv9SMFVt37qoCBdqRffuuklWYQlCNnEs2ZaIBD1kZRNnikiZOS1eqgkar0iw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/vite": {
       "version": "5.4.21",
       "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz",

backend/package.json

@@ -7,9 +7,11 @@
     "prebuild": "node scripts/clean-dist.mjs",
     "build": "tsc -p tsconfig.json",
     "catalog:migrate": "node dist/catalog/migrate.js",
+    "sensitivity:shadow": "node dist/catalog/sensitivity-shadow.js",
     "test": "vitest run",
     "start": "node dist/server.js",
-    "test:schema-v3-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs"
+    "test:schema-v4-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs",
+    "test:schema-v3-verifier": "npm run test:schema-v4-verifier"
   },
   "dependencies": {
     "@fastify/cookie": "11.1.2",
@@ -18,14 +20,17 @@
     "@types/pg": "^8.20.3",
     "fastify": "^5.0.0",
     "kysely": "^0.29.5",
+    "libphonenumber-js": "1.13.12",
     "openid-client": "6.8.5",
     "pg": "^8.22.0",
+    "validator": "13.15.35",
     "yaml": "^2.9.0",
     "zod": "^4.4.3"
   },
   "devDependencies": {
     "@testcontainers/postgresql": "^12.1.0",
     "@types/node": "24.13.3",
+    "@types/validator": "13.15.10",
     "tsx": "^4.19.0",
     "typescript": "^5.6.0",
     "vitest": "^2.1.0"

backend/python/sensitivity-ner-model-sha256.txt

@@ -0,0 +1,8 @@
+34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930  ./.gitattributes
+4d9344c58a2a2ea4bb4ff4f7c611a853cf413205fc10d0cace564eba06f73828  ./README.md
+180f0a10d1d5ed5ce3318db0bcb0b1b7780d79a52f0a8fc3acbd27f74536d0e4  ./THOTHII_MODEL_REVISION
+164f17362bcf9d114067d3465e7374bfdd79ce6b605acb745de5a49dabb9595c  ./config.json
+f27dd63cc43a248d2566f0b6ad7a115db353676ce0561dcbca45bac766464c1a  ./encoder_config/config.json
+0280f6f39f6012da50b6640bad438d9b7e763a1b0102094115d1b710c4dd79b6  ./model.safetensors
+f6df10ec83bea993035b2dd7c39345a3d4fcf23421c2adb6cb4ffc1e6d1bc4b5  ./tokenizer.json
+233beed1f1095cccfc7907cde31a8d90a0c6aa4fdfaf6493f8e55fd162e81ae6  ./tokenizer_config.json

backend/python/sensitivity-ner-requirements.txt

@@ -0,0 +1,34 @@
+# Optional offline CPU pack. Fully version-locked in its own venv; not part of the base image.
+--extra-index-url https://download.pytorch.org/whl/cpu
+accelerate==1.14.0
+annotated-types==0.8.0
+certifi==2026.7.22
+charset-normalizer==3.5.1
+filelock==3.32.5
+fsspec==2026.7.0
+gliner2[local]==2.0.0
+hf-xet==1.6.0
+huggingface-hub==0.36.2
+idna==3.19
+Jinja2==3.1.6
+MarkupSafe==3.0.3
+mpmath==1.3.0
+networkx==3.6.1
+numpy==2.5.2
+packaging==26.3
+peft==0.20.0
+psutil==7.2.2
+pydantic==2.13.5
+pydantic-core==2.46.5
+PyYAML==6.0.3
+regex==2026.9.3
+requests==2.34.2
+safetensors==0.8.0
+sympy==1.14.0
+tokenizers==0.22.2
+torch==2.14.0+cpu
+tqdm==4.70.0
+transformers==4.57.6
+typing-extensions==4.16.0
+typing-inspection==0.4.4
+urllib3==2.7.0

backend/python/sensitivity_ner_worker.py

@@ -0,0 +1,301 @@
+"""Offline, CPU-only JSONL worker for optional sensitivity NER evidence."""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import ctypes
+import errno
+import hashlib
+import json
+import os
+import socket
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+PII_LABELS = [
+    "person",
+    "full_name",
+    "first_name",
+    "middle_name",
+    "last_name",
+    "date_of_birth",
+    "email",
+    "phone_number",
+    "address",
+    "street_address",
+    "city",
+    "state_or_region",
+    "postal_code",
+    "country",
+    "government_id",
+    "national_id_number",
+    "passport_number",
+    "drivers_license_number",
+    "license_number",
+    "tax_id",
+    "tax_number",
+    "bank_account",
+    "account_number",
+    "routing_number",
+    "iban",
+    "payment_card",
+    "card_number",
+    "card_expiry",
+    "card_cvv",
+    "username",
+    "ip_address",
+    "account_id",
+    "sensitive_account_id",
+    "password",
+    "secret",
+    "api_key",
+    "access_token",
+    "recovery_code",
+    "sensitive_date",
+    "document_date",
+    "expiration_date",
+    "transaction_date",
+]
+
+_MODEL_COMPAT_DIRECTORY: tempfile.TemporaryDirectory[str] | None = None
+_EXPECTED_MODEL_REVISION = "c153999da5f4c509df4322b0c6a1baf3d2c284d7"
+
+
+def _arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--threads", type=int, default=2)
+    return parser.parse_args()
+
+
+def _disable_network() -> None:
+    libc = ctypes.CDLL(None, use_errno=True)
+    libc.prctl.argtypes = [
+        ctypes.c_int,
+        ctypes.c_ulong,
+        ctypes.c_ulong,
+        ctypes.c_ulong,
+        ctypes.c_ulong,
+    ]
+    libc.prctl.restype = ctypes.c_int
+    if libc.prctl(38, 1, 0, 0, 0) != 0:  # PR_SET_NO_NEW_PRIVS
+        raise RuntimeError("cannot enable no-new-privileges for network isolation")
+
+    try:
+        seccomp = ctypes.CDLL("libseccomp.so.2", use_errno=True)
+    except OSError as error:
+        raise RuntimeError("libseccomp is required for network isolation") from error
+    seccomp.seccomp_init.argtypes = [ctypes.c_uint32]
+    seccomp.seccomp_init.restype = ctypes.c_void_p
+    seccomp.seccomp_syscall_resolve_name.argtypes = [ctypes.c_char_p]
+    seccomp.seccomp_syscall_resolve_name.restype = ctypes.c_int
+    seccomp.seccomp_rule_add.argtypes = [
+        ctypes.c_void_p,
+        ctypes.c_uint32,
+        ctypes.c_int,
+        ctypes.c_uint,
+    ]
+    seccomp.seccomp_rule_add.restype = ctypes.c_int
+    seccomp.seccomp_load.argtypes = [ctypes.c_void_p]
+    seccomp.seccomp_load.restype = ctypes.c_int
+    seccomp.seccomp_release.argtypes = [ctypes.c_void_p]
+    seccomp.seccomp_release.restype = None
+
+    allow = 0x7FFF0000  # SCMP_ACT_ALLOW
+    deny = 0x00050000 | errno.EPERM  # SCMP_ACT_ERRNO(EPERM)
+    filter_context = seccomp.seccomp_init(allow)
+    if not filter_context:
+        raise RuntimeError("cannot initialize network syscall filter")
+    try:
+        for syscall in (
+            "socket",
+            "connect",
+            "sendto",
+            "sendmsg",
+            "sendmmsg",
+            "bind",
+            "listen",
+            "accept",
+            "accept4",
+        ):
+            syscall_number = seccomp.seccomp_syscall_resolve_name(syscall.encode("ascii"))
+            if syscall_number < 0:
+                raise RuntimeError(f"cannot resolve network syscall: {syscall}")
+            if seccomp.seccomp_rule_add(filter_context, deny, syscall_number, 0) != 0:
+                raise RuntimeError(f"cannot block network syscall: {syscall}")
+        if seccomp.seccomp_load(filter_context) != 0:
+            raise RuntimeError("cannot activate network syscall filter")
+    finally:
+        seccomp.seccomp_release(filter_context)
+
+    def blocked(*_args: Any, **_kwargs: Any) -> Any:
+        raise PermissionError(errno.EPERM, "network disabled")
+
+    socket.socket = blocked  # type: ignore[assignment]
+    socket.create_connection = blocked  # type: ignore[assignment]
+
+
+def _verify_model(path: Path) -> None:
+    revision_path = path / "THOTHII_MODEL_REVISION"
+    try:
+        revision = revision_path.read_text(encoding="utf-8").strip()
+    except OSError as error:
+        raise RuntimeError("model revision marker is unavailable") from error
+    if revision != _EXPECTED_MODEL_REVISION:
+        raise RuntimeError("model revision is not approved")
+
+    manifest_path = Path(__file__).with_name("sensitivity-ner-model-sha256.txt")
+    try:
+        manifest = manifest_path.read_text(encoding="utf-8").splitlines()
+    except OSError as error:
+        raise RuntimeError("model checksum manifest is unavailable") from error
+    for line in manifest:
+        checksum, separator, relative_name = line.partition("  ")
+        if not separator or len(checksum) != 64 or not relative_name.startswith("./"):
+            raise RuntimeError("model checksum manifest is invalid")
+        relative_path = Path(relative_name[2:])
+        if relative_path.is_absolute() or ".." in relative_path.parts:
+            raise RuntimeError("model checksum path is invalid")
+        model_file = path / relative_path
+        if not model_file.is_file() or model_file.is_symlink():
+            raise RuntimeError("approved model file is unavailable")
+        digest = hashlib.sha256()
+        with model_file.open("rb") as stream:
+            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+                digest.update(chunk)
+        if digest.hexdigest() != checksum:
+            raise RuntimeError("approved model checksum does not match")
+
+
+def _transformers4_model_path(path: Path) -> Path:
+    """Adapt tokenizer metadata emitted by Transformers 5 without changing pinned weights.
+
+    GLiNER2 2.0.0 officially requires Transformers <5, while current Fastino checkpoints were
+    saved by Transformers 5.8.0.  Transformers 4 calls the same list
+    ``additional_special_tokens``; Transformers 5 renamed it to ``extra_special_tokens`` and
+    changed its type.  Keep the downloaded model immutable and create a temporary symlink view
+    containing only the compatibility metadata needed by the supported GLiNER2 dependency set.
+    """
+
+    tokenizer_path = path / "tokenizer_config.json"
+    try:
+        tokenizer = json.loads(tokenizer_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise RuntimeError("invalid tokenizer configuration") from error
+    extra_tokens = tokenizer.get("extra_special_tokens")
+    if extra_tokens is None:
+        return path
+    if not isinstance(extra_tokens, list) or not all(isinstance(token, str) for token in extra_tokens):
+        raise RuntimeError("unsupported extra_special_tokens configuration")
+    if "additional_special_tokens" in tokenizer:
+        raise RuntimeError("ambiguous special-token configuration")
+
+    global _MODEL_COMPAT_DIRECTORY
+    _MODEL_COMPAT_DIRECTORY = tempfile.TemporaryDirectory(prefix="thothii-ner-model-")
+    compatible_path = Path(_MODEL_COMPAT_DIRECTORY.name)
+    for child in path.iterdir():
+        if child.name == tokenizer_path.name:
+            continue
+        (compatible_path / child.name).symlink_to(child, target_is_directory=child.is_dir())
+    tokenizer["additional_special_tokens"] = tokenizer.pop("extra_special_tokens")
+    (compatible_path / tokenizer_path.name).write_text(
+        json.dumps(tokenizer, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    return compatible_path
+
+
+def _load_model(model_path: str, threads: int) -> Any:
+    path = Path(model_path).resolve(strict=True)
+    if not path.is_dir():
+        raise RuntimeError("model path must be a local directory")
+    _verify_model(path)
+    os.environ["CUDA_VISIBLE_DEVICES"] = ""
+    os.environ["HIP_VISIBLE_DEVICES"] = ""
+    os.environ["HF_HUB_OFFLINE"] = "1"
+    os.environ["TRANSFORMERS_OFFLINE"] = "1"
+    import torch
+    from gliner2 import AutoExtractor
+
+    torch.set_num_threads(max(1, min(threads, 8)))
+    torch.set_num_interop_threads(1)
+    compatible_path = _transformers4_model_path(path)
+    with contextlib.redirect_stdout(sys.stderr):
+        model = AutoExtractor.from_pretrained(str(compatible_path), map_location="cpu")
+    _disable_network()
+    return model
+
+
+def _request(value: Any) -> tuple[str, list[dict[str, str]]]:
+    if not isinstance(value, dict) or not isinstance(value.get("id"), str):
+        raise ValueError("invalid request")
+    candidates = value.get("candidates")
+    if not isinstance(candidates, list) or not 1 <= len(candidates) <= 128:
+        raise ValueError("invalid candidates")
+    parsed: list[dict[str, str]] = []
+    for candidate in candidates:
+        if not isinstance(candidate, dict):
+            raise ValueError("invalid candidate")
+        column_id = candidate.get("columnId")
+        text = candidate.get("text")
+        if not isinstance(column_id, str) or not isinstance(text, str) or not 1 <= len(text) <= 500:
+            raise ValueError("invalid candidate")
+        parsed.append({"columnId": column_id, "text": text})
+    return value["id"], parsed
+
+
+def _detect(model: Any, candidates: list[dict[str, str]]) -> list[dict[str, Any]]:
+    evidence: list[dict[str, Any]] = []
+    for candidate in candidates:
+        result = model.extract_entities(
+            candidate["text"],
+            PII_LABELS,
+            threshold=0.5,
+            include_confidence=True,
+        )
+        entities = result.get("entities", {}) if isinstance(result, dict) else {}
+        best: tuple[str, float] | None = None
+        if isinstance(entities, dict):
+            for label, matches in entities.items():
+                if label not in PII_LABELS or not isinstance(matches, list):
+                    continue
+                for match in matches:
+                    if not isinstance(match, dict):
+                        continue
+                    confidence = match.get("confidence")
+                    if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
+                        continue
+                    if best is None or confidence > best[1]:
+                        best = (label, float(confidence))
+        if best is not None:
+            evidence.append(
+                {
+                    "columnId": candidate["columnId"],
+                    "label": best[0],
+                    "confidence": best[1],
+                }
+            )
+    return evidence
+
+
+def main() -> int:
+    args = _arguments()
+    model = _load_model(args.model, args.threads)
+    print(json.dumps({"ready": True}, separators=(",", ":")), flush=True)
+    for line in sys.stdin:
+        request_id = "invalid"
+        try:
+            request_id, candidates = _request(json.loads(line))
+            response = {"id": request_id, "ok": True, "evidence": _detect(model, candidates)}
+        except Exception:
+            response = {"id": request_id, "ok": False, "error": "detection_failed"}
+        print(json.dumps(response, separators=(",", ":")), flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/scripts/p1-acceptance.mjs

@@ -1195,13 +1195,8 @@ export async function executeChecks({ checks, failAt, recorder } = {}) {
 
 function baseWorkspace(id, evidenceSource) {
   return {
-    workspace: { schema_version: 3, id, name: `P1 ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P1 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
   };
 }

backend/scripts/p1-manual-acceptance.mjs

@@ -109,7 +109,7 @@ async function validateDistFiles(repo,files){const dist=join(repo,"backend","dis
 
 export async function readManualOwnership({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),root=fixedManualRoot(repo);noSymlinkExisting(repo,root);let rootEntry,ownershipEntry;try{rootEntry=await lstat(root);ownershipEntry=await lstat(join(root,"ownership.json"));}catch{throw new Error("manual ownership is missing");}if(!rootEntry.isDirectory()||rootEntry.isSymbolicLink()||await realpath(root)!==root||!ownershipEntry.isFile()||ownershipEntry.isSymbolicLink())throw new Error("manual ownership is unsafe");let value;try{value=JSON.parse(await readFile(join(root,"ownership.json"),"utf8"));}catch{throw new Error("manual ownership is malformed");}const baseValid=value.schemaVersion===1&&value.kind==="p1-manual-acceptance"&&HEX64.test(value.nonce??"")&&value.repositoryRoot===repo&&value.root===root&&value.status==="PENDING"&&["PREPARING","READY"].includes(value.stage)&&value.listener?.host===HOST&&value.listener?.port===PORT&&value.listener?.state==="stopped"&&typeof value.createdAt==="string"&&validEntrypoint(value.entrypoint,repo)&&validDistManifest(value.distManifest,root)&&JSON.stringify(value.resources)===JSON.stringify([root,{kind:"fastify",host:HOST,port:PORT}]);const readyLog=value.backendLog?.path===join(root,"logs/backend.log")&&Number.isSafeInteger(value.backendLog?.dev)&&Number.isSafeInteger(value.backendLog?.ino);if(!baseValid||(value.stage==="READY"?!readyLog:value.backendLog!==null))throw new Error("manual ownership identity mismatch");return value;}
 async function run(executable,argv,options={}){return await exec(executable,argv,{...options,maxBuffer:2*1024*1024,encoding:"utf8"});}
-function descriptor(id,source){return{workspace:{schema_version:3,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},semantic_index:{vector_store:{engine:"qdrant",collection:id,dimensions:1024,distance:"cosine"},embedding:{provider:"ollama_internal",model:"qwen3-embedding:0.6b",dimensions:1024}},llm_policy:{allowed:["zai/glm-5.2"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
+function descriptor(id,source){return{workspace:{schema_version:4,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
 function descriptors(){return[descriptor("p1-filesystem",{type:"filesystem",uri:"workspace-content/p1-filesystem/evidence",patterns:["**/*.md"],max_bytes:10485760}),descriptor("p1-http",{type:"http",uris:["https://evidence.example.test/guide.md"],authentication:"signed_urls_file",connect_timeout_ms:1250,read_timeout_ms:30001,max_bytes:12345,max_redirects:2,allow_private_hosts:false,max_cache_bytes:67890}),descriptor("p1-s3",{type:"s3",uri:"s3://p1-evidence/published/",endpoint_url:"https://s3.example.test/",region:"eu-west-1",credentials:"static_files",trusted_endpoint:true,allow_private_endpoint:false,allow_insecure_endpoint:false,max_bytes:12345,max_objects:33,max_pages:4,page_size:5})];}
 function quote(value){return `'${String(value).replaceAll("'",`'"'"'`)}'`;}
 async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof","python3"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}}

backend/scripts/p1-manual-acceptance.test.mjs

@@ -403,7 +403,7 @@ test("generated render command validates saved responses and owned snapshot befo
 });
 
 const renderSnapshotYaml=`workspace:
-  schema_version: 3
+  schema_version: 4
   id: p1-filesystem
   name: P1 filesystem
   language: en
@@ -412,11 +412,6 @@ dwh:
   database: postgres
   schema: public
   supported_transports: [postgres_direct]
-semantic_index:
-  vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
-  embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
-llm_policy:
-  allowed: [zai/glm-5.2]
 evidence:
   source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
   policy: {max_chunk_chars: 4000, retain_published_generations: 3}

backend/scripts/p1-render-snapshot.test.mjs

@@ -16,7 +16,7 @@ async function fixture() {
  await writeFile(join(root,"installation/base.yaml"),"{}\n");
  const secret=join(root,"fixture-secrets/dwh-password"); await writeFile(secret,"not-inspected",{mode:0o600});
  await writeFile(snapshot,`workspace:
-  schema_version: 3
+  schema_version: 4
   id: p1-filesystem
   name: P1 filesystem
   language: en
@@ -25,11 +25,6 @@ dwh:
   database: postgres
   schema: public
   supported_transports: [postgres_direct]
-semantic_index:
-  vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
-  embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
-llm_policy:
-  allowed: [zai/glm-5.2]
 evidence:
   source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
   policy: {max_chunk_chars: 4000, retain_published_generations: 3}
@@ -61,7 +56,7 @@ test("renderer refuses snapshot manifest head, digest, and expected-digest tampe
 
 test("renderer refuses a missing or malformed snapshot manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/nomanifest.yaml"); await rm(f.manifestPath); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*(missing|unbounded|unsafe)/); await writeFile(f.manifestPath,"{not json"); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*malformed/); await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); });
 
-test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n  schema_version: 3\n  id: p1-filesystem\n  name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); });
+test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n  schema_version: 4\n  id: p1-filesystem\n  name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); });
 
 test("renderer anchors publication when rendered parent is concurrently swapped", async()=>{
  const f=await fixture(),output=join(f.root,"rendered/raced.yaml"),moved=join(f.root,"rendered-moved"),outside=join(f.repo,"outside-rendered"); await mkdir(outside);

backend/scripts/p11-acceptance.mjs

@@ -328,13 +328,8 @@ async function tht(ctx, argv, options = {}) {
 function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
 function baseWorkspace(id, evidenceSource) {
   return {
-    workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
   };
 }

backend/scripts/p11-manual-acceptance.mjs

@@ -81,13 +81,8 @@ async function git(executable, argv, options = {}) {
 function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
 function baseWorkspace(id, evidenceSource) {
   return {
-    workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
   };
 }

backend/scripts/p2-acceptance.mjs

@@ -375,16 +375,11 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p2p6-acceptance.mjs

@@ -376,16 +376,11 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p3-acceptance.mjs

@@ -379,16 +379,11 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p5-acceptance.mjs

@@ -374,16 +374,11 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p6-acceptance.mjs

@@ -374,16 +374,11 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
-    semantic_index: {
-      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
-      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
-    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/verify-workspace-descriptor-files.mjs

@@ -22,6 +22,7 @@ const reviewedExpandableBlocks = new Map([
     { sha256: "37f18ce7ce93cb8b84f3b3708462cc16d50fdc7bab22836c382dbacf8382f05f", rationale: "Generates the reviewed synthetic tht installer artifact." },
   ]],
   ["scripts/test-server-pi-state-topology.sh", [
+    { sha256: "435c769b8cbd7b834f56fdabddb86ba04fb404dd0d8a6b7c21719a8b0f7cf011", rationale: "Generates the reviewed model-catalog projection override for the isolated server topology test." },
     { sha256: "6ae9567db53d6cd45a2c19c98acaf45f382450b157ea7d6f6d35125f68c50947", rationale: "Generates the isolated server topology test environment, including its installation descriptor and authentication configuration root." },
   ]],
   ["scripts/test-vector-backup-restore-safety.sh", [
@@ -36,11 +37,10 @@ const reviewedExpandableBlocks = new Map([
     { sha256: "b903e5dae953ae1372f1a5276f12a92ed3dd632b897f3afe5e00c646d90a1b42", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
   ]],
   ["scripts/unified-deployment-smoke.sh", [
-    { sha256: "1d60bf140165a8fabfa0c3729e776136904717e67becf3e0ab68c70d8e37847e", rationale: "Generates reviewed Task 13 runtime configuration." },
-    { sha256: "36d3d8a2362dbdc4fad90948d6c227586d749f56b9a4bc5b6b5a91bcbec6407b", rationale: "Generates the reviewed local Task 13 Compose override." },
-    { sha256: "c556f7d910d0788e219b042957e6b307cb9925b43920c680535d0d3a6dcbdb25", rationale: "Generates the reviewed local Task 13 installation descriptor." },
+    { sha256: "b6c0826151b2c8b955399d1abf5b691cc8fe6b6454b17da000dde7ba3bc55d2d", rationale: "Generates the reviewed local Task 13 Compose override with normalized catalog mounts." },
+    { sha256: "24f69d12b8554aa2bebba455be99fde3e60743eef5a40fa2ef5b29397a477c03", rationale: "Generates the reviewed local Task 13 installation descriptor with its model catalog." },
     { sha256: "526006fa6d48a8080b3834723630c64de5005a67243e944ebf1da15212b4d654", rationale: "Generates the reviewed server Task 13 Compose override." },
-    { sha256: "c57ae2205c21ead0c2015a353aaabb948fa4ddd9b78a2cdcdb71f48cf2db742d", rationale: "Generates the reviewed projected-auth server Task 13 installation descriptor." },
+    { sha256: "406ccead1967f642225c946fc4a23fe5b019c9764cc5153e1125876ade16ec90", rationale: "Generates the reviewed projected-auth server Task 13 installation descriptor with its model catalog." },
   ]],
   ["scripts/vector-backup.sh", [
     { sha256: "571899db49dfdcec8107fbe1e0a86a61e7581979d3c4c248c20546843e275bcf", rationale: "Generates the reviewed backup manifest inside the helper command." },
@@ -103,6 +103,7 @@ function isPolicyImplementationException(label, category) {
   ]);
   if (implementations.has(label)) return true;
   if (category === "migration-marker" && new Set([
+    "backend/src/workspaces/schema.ts",
     "scripts/workspace_descriptor_doc_contract.py",
     "scripts/test_workspace_descriptor_doc_contract.py",
     "backend/scripts/clean-dist.test.mjs",
@@ -173,7 +174,7 @@ function validateWorkspaceSource(source, label, { requireWorkspace, expandable =
   try {
     parseWorkspaceYaml(source);
   } catch (error) {
-    throw new Error(`${label}: workspace descriptor is not valid schema v3: ${error instanceof Error ? error.message : String(error)}`);
+    throw new Error(`${label}: workspace descriptor is not valid schema v4: ${error instanceof Error ? error.message : String(error)}`);
   }
   return true;
 }

backend/scripts/verify-workspace-descriptor-files.test.mjs

@@ -33,20 +33,20 @@ function bashN(root, path) {
 function replaceWorkspaceKeys(source, workspaceKey, schemaLine) {
   return source
     .replace(/^workspace:$/m, workspaceKey)
-    .replace(/^  schema_version: 3$/m, schemaLine);
+    .replace(/^  schema_version: 4$/m, schemaLine);
 }
 
-test("production parser accepts semantic v3 with quoted Unicode/tagged keys and spacing", async (t) => {
+test("production parser accepts semantic v4 with quoted Unicode/tagged keys and spacing", async (t) => {
   const root = await fixture(t);
   const unicode = replaceWorkspaceKeys(
     canonicalDescriptor,
     '"\\u0077orkspace" :',
-    '  "\\u0073chema_version" : 3',
+    '  "\\u0073chema_version" : 4',
   );
   const tagged = replaceWorkspaceKeys(
     canonicalDescriptor,
     "!!str workspace :",
-    "  !!str schema_version : 3",
+    "  !!str schema_version : 4",
   );
   await put(root, "deploy/workspaces/unicode.yaml", unicode);
   await put(root, "deploy/workspaces/tagged.yaml", tagged);
@@ -59,14 +59,15 @@ test("production parser accepts semantic v3 with quoted Unicode/tagged keys and
   });
 });
 
-test("production parser rejects fancy keys with every non-v3 or ambiguous value", async (t) => {
+test("production parser rejects fancy keys with every non-v4 or ambiguous value", async (t) => {
   const invalid = [
     ["unicode-v2", '"\\u0077orkspace" :', '  "\\u0073chema_version" : 2'],
-    ["tagged-leading-zero", "!!str workspace :", "  !!str schema_version : 02"],
-    ["hexadecimal", "workspace :", "  schema_version : 0x2"],
-    ["multiline", "workspace :", "  schema_version : >\n    3"],
-    ["duplicate", "workspace :", "  schema_version : 3\n  schema_version: 3"],
-    ["inline", "workspace: { schema_version: 3 }", "  schema_version: 3"],
+    ["unicode-v3", '"\\u0077orkspace" :', '  "\\u0073chema_version" : 3'],
+    ["tagged-leading-zero", "!!str workspace :", "  !!str schema_version : 03"],
+    ["hexadecimal", "workspace :", "  schema_version : 0x3"],
+    ["multiline", "workspace :", "  schema_version : >\n    4"],
+    ["duplicate", "workspace :", "  schema_version : 4\n  schema_version: 4"],
+    ["inline", "workspace: { schema_version: 4 }", "  schema_version: 4"],
   ];
   for (const [name, workspaceKey, schemaLine] of invalid) {
     await t.test(name, async () => {
@@ -120,7 +121,7 @@ test("PowerShell embedded workspace mappings are rejected while bundle-only stri
   const root = await fixture(t);
   const source = [
     "$workspace = @'",
-    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 0x2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 0x2").trimEnd(),
     "'@",
     '$bundle = @"',
     "bundle:",
@@ -137,7 +138,7 @@ test("PowerShell embedded workspace mappings are rejected while bundle-only stri
 
 test("workspace descriptor family entries require a top-level workspace", async (t) => {
   const root = await fixture(t);
-  await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n  schema_version: 3\n");
+  await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n  schema_version: 4\n");
   await assert.rejects(
     verifyEntries({
       root,
@@ -179,7 +180,7 @@ test("script scalar workspace remains a bundle even with descriptor-like sibling
 test("standalone descriptor files require workspace to be a mapping", async (t) => {
   const root = await fixture(t);
   const path = "scripts/fixtures/workspace-registry-scalar.yaml";
-  await put(root, path, "workspace: analytics\nschema_version: 3\n");
+  await put(root, path, "workspace: analytics\nschema_version: 4\n");
   await assert.rejects(
     verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }),
     /workspace.*mapping/i,
@@ -193,11 +194,11 @@ test("Bash extractor supports hyphen, digit, escaped delimiters, and tab strippi
       name: "hyphen-v2",
       opener: "cat <<'WORKSPACE-YAML'",
       delimiter: "WORKSPACE-YAML",
-      descriptor: canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2"),
+      descriptor: canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2"),
       rejected: true,
     },
     {
-      name: "digit-v3",
+      name: "digit-v4",
       opener: "cat <<2YAML",
       delimiter: "2YAML",
       descriptor: canonicalDescriptor,
@@ -207,11 +208,11 @@ test("Bash extractor supports hyphen, digit, escaped delimiters, and tab strippi
       name: "escaped-v2",
       opener: "cat <<WORKSPACE\\-YAML",
       delimiter: "WORKSPACE-YAML",
-      descriptor: canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2"),
+      descriptor: canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2"),
       rejected: true,
     },
     {
-      name: "tab-strip-v3",
+      name: "tab-strip-v4",
       opener: "cat <<-'TAB-YAML'",
       delimiter: "\tTAB-YAML",
       descriptor: canonicalDescriptor.split("\n").map((line) => `\t${line}`).join("\n"),
@@ -272,7 +273,7 @@ test("non-stripping heredoc close requires an exact physical delimiter line", as
     "#!/usr/bin/env bash",
     "cat <<'---'",
     "---   ",
-    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
     "---",
     "",
   ].join("\n");
@@ -313,7 +314,7 @@ test("double-quoted non-special backslash is preserved in the delimiter", async
     "#!/usr/bin/env bash",
     'cat <<"\\---"',
     "---",
-    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
     "\\---",
     "",
   ].join("\n");
@@ -355,7 +356,7 @@ test("split heredoc operator continuation cannot bypass v2 validation", async (t
     "#!/usr/bin/env bash",
     "cat <\\",
     "<'YAML'",
-    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
     "YAML",
     "",
   ].join("\n");
@@ -424,7 +425,7 @@ test("PowerShell comment backslash cannot hide a following v2 here-string", asyn
   const source = [
     "# harmless PowerShell comment \\",
     "$workspace = @'",
-    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
     "'@",
     "",
   ].join("\n");
@@ -435,7 +436,7 @@ test("PowerShell comment backslash cannot hide a following v2 here-string", asyn
   );
 });
 
-test("PowerShell dialect accepts normal v3 and non-workspace bundle here-strings", async (t) => {
+test("PowerShell dialect accepts normal v4 and non-workspace bundle here-strings", async (t) => {
   const root = await fixture(t);
   const path = "scripts/powershell-valid-smoke.ps1";
   const source = [
@@ -494,10 +495,10 @@ test("PowerShell cast and concatenation openers cannot hide embedded descriptors
 test("expandable YAML interpolation that can hide a workspace descriptor fails closed", async (t) => {
   const root = await fixture(t);
   const cases = [
-    ["braced-key", "${key}:\n  schema_version: 3"],
-    ["plain-key", "$key:\n  schema_version: 3"],
-    ["quoted-key", '"$key" :\n  schema_version: 3'],
-    ["subexpression-key", "$($key):\n  schema_version: 3"],
+    ["braced-key", "${key}:\n  schema_version: 4"],
+    ["plain-key", "$key:\n  schema_version: 4"],
+    ["quoted-key", '"$key" :\n  schema_version: 4'],
+    ["subexpression-key", "$($key):\n  schema_version: 4"],
     ["version", "workspace:\n  schema_version: $version"],
   ];
   for (const [name, body] of cases) {
@@ -564,7 +565,7 @@ test("unmarked expandable Bash YAML cannot generate descriptor keys or values at
       "key=workspace",
       "cat <<YAML",
       generatedKey,
-      "  schema_version: 3",
+      "  schema_version: 4",
       "YAML",
       "",
     ].join("\n");
@@ -600,14 +601,14 @@ test("an in-band marker cannot authorize expandable content", async (t) => {
   for (const [path, source] of [
     ["scripts/fake-marker.sh", [
       "#!/usr/bin/env bash",
-      "# schema-v3-only: expandable-nonworkspace",
+      "# schema-v4-only: expandable-nonworkspace",
       "cat <<YAML",
       "${DESCRIPTOR}",
       "YAML",
       "",
     ].join("\n")],
     ["scripts/fake-marker.ps1", [
-      "# schema-v3-only: expandable-nonworkspace",
+      "# schema-v4-only: expandable-nonworkspace",
       '$yaml = @"',
       "$descriptor",
       '"@',

backend/src/app.ts

@@ -3,9 +3,11 @@ import cors from "@fastify/cors";
 import cookie from "@fastify/cookie";
 import rateLimit from "@fastify/rate-limit";
 import { dirname, isAbsolute, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { tmpdir } from "node:os";
 import type { AppConfig } from "./config.js";
 import { ThtRunner } from "./tht/tht-runner.js";
+import { createMemoryCleanup } from "./catalog/memory-cleanup.js";
 import { PiProcessManager } from "./pi/pi-process-manager.js";
 import { SseHub } from "./sse/sse-hub.js";
 import { authenticateSession, captureAuthConfigSnapshot, configuredOrigin } from "./auth/auth.js";
@@ -22,7 +24,8 @@ import { isUsableAuthenticationSecret } from "./auth/secret-policy.js";
 import { secretValue } from "./config/secret-bundle.js";
 import { sessionRoutes } from "./routes/sessions.js";
 import { sqlRoutes } from "./routes/sql.js";
-import { metaRoutes, type ListModelsFn } from "./routes/meta.js";
+import { metaRoutes } from "./routes/meta.js";
+import type { ListModelsFn } from "./pi/list-models.js";
 import { settingsRoutes, effectiveSettings } from "./routes/settings.js";
 import { createPiModelLister } from "./pi/list-models.js";
 import { createPiManagement, type PiManagementService } from "./pi/management.js";
@@ -31,7 +34,11 @@ import { ReadinessManager } from "./runtime/readiness-manager.js";
 import { MaintenanceBarrier } from "./runtime/maintenance-gate.js";
 import { WorkspaceRegistry } from "./workspaces/registry.js";
 import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js";
-import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js";
+import {
+  workspaceRoutes,
+  type WorkspaceDatabaseTester,
+  type WorkspaceDiagnoser,
+} from "./routes/workspaces.js";
 import { piManagementRoutes } from "./routes/pi-management.js";
 import { supportsSessionRuntime } from "./workspaces/bindings.js";
 import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js";
@@ -56,8 +63,11 @@ import { metadataGenerationModelRoutes } from "./routes/metadata-generation-mode
 import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-description-consolidation.js";
 import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js";
 import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js";
-import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js";
-import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js";
+import { SensitivityAnalysisService } from "./catalog/sensitivity-analysis-service.js";
+import { SensitivityAnalysisRunner } from "./catalog/sensitivity-analysis-runner.js";
+import { SensitivityClassifier, type LocalNerDetector, type SensitivityValueSource } from "./catalog/sensitivity-classifier.js";
+import { ConcreteSensitivityValueSource } from "./catalog/sensitivity-value-source.js";
+import { PythonLocalNerDetector } from "./catalog/local-ner-detector.js";
 import {
   ConcreteDescriptionSourceSampler,
   type DescriptionSourceSampler,
@@ -66,6 +76,13 @@ import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description
 import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js";
 import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
 import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
+import { loadRuntimeModelCatalog, type RuntimeModelCatalog } from "./models/runtime-model-catalog.js";
+import { createProductionWorkspacePreprocessingService } from "./workspace-maintenance.js";
+import type { WorkspacePreprocessingService } from "./workspaces/preprocessing-service.js";
+import { PreprocessingStateStore } from "./workspaces/preprocessing-state.js";
+import { workspacePreprocessingRoutes } from "./routes/workspace-preprocessing.js";
+import { memoryRoutes } from "./routes/memory.js";
+import { evidenceRoutes } from "./routes/evidence.js";
 
 export interface BuildAppDeps {
   thtRunner?: ThtRunner;
@@ -77,7 +94,9 @@ export interface BuildAppDeps {
   hub?: SseHub;
   workspaceRegistry?: WorkspaceRegistry;
   workspaceDiagnoser?: WorkspaceDiagnoser;
+  workspaceDatabaseTester?: WorkspaceDatabaseTester;
   workspaceSecretStore?: WorkspaceSecretStore;
+  workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear"> & Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
   catalogRepository?: CatalogRepository;
   catalogService?: CatalogService;
   catalogPostgresAccess?: CatalogPostgresAccess;
@@ -88,8 +107,11 @@ export interface BuildAppDeps {
   catalogSyncWorker?: CatalogSyncWorker;
   catalogOperationCoordinator?: CatalogOperationCoordinator;
   metadataGenerationModels?: MetadataGenerationModels;
+  runtimeModelCatalog?: RuntimeModelCatalog;
   modelCompleter?: ModelCompleter;
   descriptionSourceSampler?: DescriptionSourceSampler;
+  sensitivityValueSource?: SensitivityValueSource;
+  localNerDetector?: LocalNerDetector;
   workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
   maintenanceBarrier?: MaintenanceBarrier;
   piManagement?: PiManagementService;
@@ -142,6 +164,8 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   app.register(cookie);
   app.register(rateLimit, { global: false });
 
+  const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
+  const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
   const tht = deps?.thtRunner ?? new ThtRunner({
     thtBin: config.thtBin,
     harnessDir: config.harnessDir,
@@ -152,20 +176,32 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     secretsFile: config.secretsFile,
     secretFiles: config.secretFiles,
     workspaceSecretStore,
+    catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
     semanticRuntime: {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
+      internalEmbeddingId: config.internalEmbeddingId,
       internalEmbeddingModel: config.internalEmbeddingModel,
       internalEmbeddingDimensions: config.internalEmbeddingDimensions,
     },
   });
-  const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
+  const workspacePreprocessingService = deps?.workspacePreprocessingService
+    ?? createProductionWorkspacePreprocessingService({
+      config,
+      catalogRepository,
+      registry: workspaceRegistry,
+      workspaceSecretStore,
+      runner: tht as ThtRunner,
+    });
   const hub = deps?.hub ?? new SseHub();
-  const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
-  const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
   const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
+  const runtimeModelCatalog = deps?.runtimeModelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
+  const mgr = deps?.mgr ?? new PiProcessManager(config, {
+    ...(deps?.spawnFn ? { spawnFn: deps.spawnFn } : {}),
+    modelCatalog: runtimeModelCatalog,
+  });
   const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
-    installationFile: config.installationConfigFile,
+    catalogFile: config.modelCatalogFile,
     secretsFile: config.secretsFile,
   });
   const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
@@ -186,12 +222,24 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     catalogOperationCoordinator,
     descriptionSourceSampler,
   );
-  const sensitiveDataSuggester = new SensitiveDataSuggester(
+  const sensitivityValueSource = deps?.sensitivityValueSource
+    ?? new ConcreteSensitivityValueSource(catalogPostgresAccess, workspaceSecretStore);
+  const configuredNerWorker = config.sensitivityNer?.workerScript
+    ?? fileURLToPath(new URL("../python/sensitivity_ner_worker.py", import.meta.url));
+  const localNerDetector = deps?.localNerDetector ?? (config.sensitivityNer
+    ? new PythonLocalNerDetector({
+      pythonExecutable: config.sensitivityNer.pythonExecutable,
+      workerScript: configuredNerWorker,
+      modelPath: config.sensitivityNer.modelPath,
+      cwd: dirname(configuredNerWorker),
+      threads: config.sensitivityNer.threads,
+    })
+    : undefined);
+  const sensitiveDataSuggester = new SensitivityAnalysisService(
     catalogRepository,
-    metadataGenerationModels,
-    modelCompleter,
+    new SensitivityClassifier(sensitivityValueSource, localNerDetector),
   );
-  const sensitiveDataSuggestionRunner = new SensitiveDataSuggestionRunner(
+  const sensitivityAnalysisRunner = new SensitivityAnalysisRunner(
     catalogRepository,
     sensitiveDataSuggester,
   );
@@ -204,17 +252,13 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     catalogPostgresAccess,
     catalogOperationCoordinator,
   );
+  const workspaceDatabaseTester = deps?.workspaceDatabaseTester ?? (async (workspaceId: string) => {
+    const database = await catalogRepository.getByWorkspace(workspaceId);
+    return database ? catalogService.test(database) : undefined;
+  });
   const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
   const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
     ?? new CatalogLogicalRelationshipService(catalogRepository);
-  const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider
-    ?? (config.catalogDatabase === undefined
-      ? undefined
-      : new EffectiveRelationshipSnapshotProvider(
-        catalogRepository,
-        catalogLogicalRelationshipService,
-        catalogOperationCoordinator,
-      ));
   const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
     catalogPostgresAccess,
     workspaceSecretStore,
@@ -224,19 +268,33 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     catalogSchemaIntrospector,
     catalogOperationCoordinator,
     config.catalogSyncTimeoutMs,
+    createMemoryCleanup(tht as ThtRunner, {
+      internalQdrantUrl: config.internalQdrantUrl, internalEmbeddingUrl: config.internalEmbeddingUrl,
+      internalEmbeddingId: config.internalEmbeddingId, internalEmbeddingModel: config.internalEmbeddingModel,
+      internalEmbeddingDimensions: config.internalEmbeddingDimensions,
+    }),
   );
   app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
   app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
-  app.addHook("onReady", async () => { await sensitiveDataSuggestionRunner.initialize(); });
+  app.addHook("onReady", async () => { await sensitivityAnalysisRunner.initialize(); });
+  if (localNerDetector?.warmup) {
+    app.addHook("onReady", async () => {
+      void localNerDetector.warmup?.().catch(() => undefined);
+    });
+  }
   if (!deps?.catalogRepository && catalogRepository.close) {
     app.addHook("onClose", async () => { await catalogRepository.close?.(); });
   }
   app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
   app.addHook("onClose", async () => { await descriptionGenerationWorker.stop(); });
+  if (localNerDetector?.close) {
+    app.addHook("onClose", async () => { await localNerDetector.close?.(); });
+  }
   const workspaceDiagnoser = deps?.workspaceDiagnoser
     ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
+      internalEmbeddingId: config.internalEmbeddingId,
       internalEmbeddingModel: config.internalEmbeddingModel,
       internalEmbeddingDimensions: config.internalEmbeddingDimensions,
     });
@@ -259,6 +317,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   );
 
   const listModels = deps?.listModels ?? createPiModelLister(config, {
+    modelCatalog: runtimeModelCatalog,
     warn: (detail) => app.log.warn(
       { component: "pi-model-list", detail },
       "Pi enabled-model configuration warning",
@@ -271,7 +330,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   const getSettings = async (principal: PrincipalContext): Promise<Settings> => {
     if (deps?.getSettings) return await deps.getSettings(principal);
     const stored = loadSettings(config);
-    const effective = effectiveSettings(config, stored);
+    const effective = effectiveSettings(config, stored, runtimeModelCatalog);
     // In the registry system the legacy `harness/workspaces/*.yaml` default is obsolete: when no
     // installation workspace is pinned, default to the first active registry workspace.
     if (!stored.workspace) {
@@ -284,7 +343,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     }
     return effective;
   };
-  const piManagement = deps?.piManagement ?? createPiManagement(config, { listModels });
+  const piManagement = deps?.piManagement ?? createPiManagement(config, {
+    modelCatalog: runtimeModelCatalog,
+  });
 
   const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile);
   const localRegistryResolver = deps?.localUserRegistry === undefined
@@ -408,8 +469,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     dwhPrecheck: config.dwhPrecheck,
     legacyWorkspaceMode: config.legacyWorkspaceMode,
     workspaceRuntimeSupport,
+    modelCatalog: runtimeModelCatalog,
     maintenanceBarrier,
-    effectiveRelationships,
+    catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
   });
   app.post("/internal/maintenance/activate", async (req, reply) => {
     try {
@@ -439,13 +501,33 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     return maintenanceBarrier.status();
   });
   sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
-  metaRoutes(app, { harnessDir: config.harnessDir, listModels });
+  memoryRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry, runtime: {
+    internalQdrantUrl: config.internalQdrantUrl,
+    internalEmbeddingUrl: config.internalEmbeddingUrl,
+    internalEmbeddingModel: config.internalEmbeddingModel,
+    internalEmbeddingDimensions: config.internalEmbeddingDimensions,
+  } });
+  metaRoutes(app, { harnessDir: config.harnessDir, modelCatalog: runtimeModelCatalog });
+  evidenceRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry,
+    registryRoot: config.workspaceRegistry.root, hostRegistryRoot: config.evidenceHostRegistryRoot,
+    service: workspacePreprocessingService });
   workspaceRoutes(app, {
     registry: workspaceRegistry,
     config: config.workspaceRegistry,
     diagnose: workspaceDiagnoser,
     authDiagnoser,
     secretStore: workspaceSecretStore,
+    testDatabaseConnection: workspaceDatabaseTester,
+  });
+  workspacePreprocessingRoutes(app, {
+    repository: catalogRepository,
+    registry: workspaceRegistry,
+    service: workspacePreprocessingService,
+    inputFingerprint: tht as ThtRunner,
+    readLatestJob: (workspaceId) => new PreprocessingStateStore({
+      dataRoot: config.dataRoot ?? "/data",
+      workspaceId,
+    }).readLatestJob(),
   });
   catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
   catalogTableRoutes(app, {
@@ -470,9 +552,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   catalogDescriptionGenerationRoutes(app, {
     repository: catalogRepository,
     worker: descriptionGenerationWorker,
-    sensitiveDataSuggestionRunner,
+    sensitivityAnalysisRunner,
   });
-  settingsRoutes(app, { cfg: config, listModels, getSettings });
+  settingsRoutes(app, { cfg: config, getSettings });
   piManagementRoutes(app, { service: piManagement });
 
   return app;

backend/src/auth/auth.ts

@@ -119,7 +119,9 @@ export function authenticateSession(deps: AuthDependencies): preHandlerHookHandl
       subject: session.subject,
       ...(session.displayName === undefined ? {} : { displayName: session.displayName }),
       roles: session.roles,
-      permissions: session.permissions,
+      // Sessions can outlive a deployment that changes the role permission catalog.
+      // resolve() has already checked validity, including current local user roles.
+      permissions: rolesToPermissions(session.roles),
       isAdmin: session.roles.includes("admin"),
     };
     if (STATE_CHANGING_METHODS.has(request.method)) {

backend/src/auth/config.ts

@@ -38,7 +38,7 @@ const MAX_MAPPED_GROUPS = 128;
 const ROLES = ["user", "admin"] as const;
 export const PERMISSION_CATALOG: readonly Permission[] = [
   "session.use", "session.read_all", "session.manage_all", "settings.manage",
-  "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
+  "workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
 ];
 
 const invalid = (): Error => new Error("authentication configuration is invalid");

backend/src/auth/session-store.ts

@@ -33,7 +33,7 @@ const EMPTY_HKDF_SALT = Buffer.alloc(0);
 const ROLES = ["user", "admin"] as const;
 const PERMISSIONS = [
   "session.use", "session.read_all", "session.manage_all", "settings.manage",
-  "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
+  "workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
 ] as const satisfies readonly Permission[];
 
 const invalid = (): Error => new Error("auth_session_store_invalid");

backend/src/auth/types.ts

@@ -5,7 +5,7 @@ export type Role = "user" | "admin";
 export type Permission =
   | "session.use" | "session.read_all" | "session.manage_all"
   | "settings.manage" | "workspace.manage" | "workspace.secrets.manage"
-  | "database.manage" | "pi.manage" | "auth.diagnostics.read";
+  | "database.manage" | "memory.manage" | "evidence.manage" | "pi.manage" | "auth.diagnostics.read";
 
 export interface AuthenticationSessionConfig {
   regularTtlSeconds: number;

backend/src/catalog/description-generation-worker.ts

@@ -59,10 +59,16 @@ const completionResponseSchema = z.object({
 class InvalidModelJsonError extends Error {}
 class InvalidModelSchemaError extends Error {}
 class MissingModelTargetsError extends Error {}
+
+interface DescriptionGenerationFailureTarget {
+  id: string;
+  reference: string;
+}
+
 class DescriptionGenerationBatchError extends Error {
   constructor(
     readonly failure: unknown,
-    readonly failedTargets: readonly { id: string; label: "Catalog Column" | "Catalog Table" }[],
+    readonly failedTargets: readonly DescriptionGenerationFailureTarget[],
   ) {
     super("description generation batch failed");
   }
@@ -138,7 +144,7 @@ type ParsedOutcome = z.infer<typeof outcomeSchema>;
 
 interface DescriptionGenerationPlan {
   columnTargets: SelectedColumnTarget[];
-  tableIds: string[];
+  tableTargets: Array<{ id: string; name: string }>;
 }
 
 interface DescriptionGenerationCounters {
@@ -164,10 +170,17 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
   };
 }
 
-function failureTarget(target: SelectedTarget) {
+function targetReference(target: SelectedTarget): string {
   return target.kind === "column"
-    ? { id: target.column.id, label: "Catalog Column" as const }
-    : { id: target.table.id, label: "Catalog Table" as const };
+    ? `Column ${JSON.stringify(`${target.table.name}.${target.column.name}`)}`
+    : `Table ${JSON.stringify(target.table.name)}`;
+}
+
+function failureTarget(target: SelectedTarget): DescriptionGenerationFailureTarget {
+  return {
+    id: target.kind === "column" ? target.column.id : target.table.id,
+    reference: targetReference(target),
+  };
 }
 
 const NON_GENERATABLE_DESCRIPTION: Record<DescriptionGenerationRun["language"], string> = {
@@ -681,7 +694,7 @@ function safeFailure(error: unknown): string {
 function batchFailureEvent(error: DescriptionGenerationBatchError): string {
   const summary = safeFailure(error);
   return error.failedTargets.length > 0
-    ? `${summary} Affected ${error.failedTargets[0]!.label} target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.id).join(", ")}.`
+    ? `${summary} Affected target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.reference).join(", ")}.`
     : summary;
 }
 
@@ -794,7 +807,7 @@ export class DescriptionGenerationWorker {
           scope === "selected_columns" ? "column" : "table",
         );
       }
-      const total = plan.columnTargets.length + plan.tableIds.length;
+      const total = plan.columnTargets.length + plan.tableTargets.length;
       if (total === 0 && (scope === "all" || scope === "missing")) {
         throw new DescriptionGenerationNoEligibleTargetsError(scope);
       }
@@ -934,12 +947,18 @@ export class DescriptionGenerationWorker {
     };
     await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
     throwIfCancelled(signal);
-    if (plan.tableIds.length > 0) {
-      const tableTargets = await this.resolveTableTargets(run.databaseId, plan.tableIds);
+    if (plan.tableTargets.length > 0) {
+      const tableTargets = await this.resolveTableTargets(
+        run.databaseId,
+        plan.tableTargets.map((target) => target.id),
+      );
       if (!tableTargets) {
         throw new DescriptionGenerationBatchError(
           new Error("selected tables changed during generation"),
-          plan.tableIds.map((id) => ({ id, label: "Catalog Table" })),
+          plan.tableTargets.map((target) => ({
+            id: target.id,
+            reference: `Table ${JSON.stringify(target.name)}`,
+          })),
         );
       }
       await this.processTargets(run, database, tableTargets, model, counters, signal);
@@ -1093,8 +1112,8 @@ export class DescriptionGenerationWorker {
           run.id,
           "info",
           outcome.outcome === "generated"
-            ? `Generated description for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`
-            : `Stored non-generatable result for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`,
+            ? `Generated description for ${targetReference(target)}.`
+            : `Stored non-generatable result for ${targetReference(target)}.`,
         );
       }
     }
@@ -1188,16 +1207,22 @@ export class DescriptionGenerationWorker {
       }
       return {
         columnTargets,
-        tableIds: tables
+        tableTargets: tables
           .filter((table) => scope === "all" || !table.generatedDescription?.trim())
-          .map((table) => table.id),
+          .map((table) => ({ id: table.id, name: table.name })),
       };
     }
     if (scope === "selected_tables") {
       const tableById = new Map(tables.map((table) => [table.id, table]));
       const selected = targetIds.map((tableId) => tableById.get(tableId));
       if (selected.some((table) => table === undefined)) return undefined;
-      return { columnTargets: [], tableIds: [...targetIds] };
+      return {
+        columnTargets: [],
+        tableTargets: (selected as CatalogTable[]).map((table) => ({
+          id: table.id,
+          name: table.name,
+        })),
+      };
     }
 
     const byId = new Map<string, SelectedColumnTarget>();
@@ -1209,7 +1234,7 @@ export class DescriptionGenerationWorker {
     const targets = targetIds.map((columnId) => byId.get(columnId));
     return targets.some((target) => target === undefined)
       ? undefined
-      : { columnTargets: targets as SelectedColumnTarget[], tableIds: [] };
+      : { columnTargets: targets as SelectedColumnTarget[], tableTargets: [] };
   }
 
   private async resolveTableTargets(

backend/src/catalog/local-ner-detector.ts

@@ -0,0 +1,254 @@
+import { randomUUID } from "node:crypto";
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+import { tmpdir } from "node:os";
+import { z } from "zod";
+import type {
+  LocalNerCandidate,
+  LocalNerDetector,
+  LocalNerEvidence,
+} from "./sensitivity-classifier.js";
+
+const MAX_LINE_BYTES = 64 * 1024;
+const candidateSchema = z.object({
+  columnId: z.uuid(),
+  text: z.string().min(1).max(500),
+}).strict();
+const workerMessageSchema = z.union([
+  z.object({ ready: z.literal(true) }).strict(),
+  z.object({
+    id: z.uuid(),
+    ok: z.literal(true),
+    evidence: z.array(z.object({
+      columnId: z.uuid(),
+      label: z.string().min(1).max(80),
+      confidence: z.number().min(0).max(1),
+    }).strict()).max(1_000),
+  }).strict(),
+  z.object({ id: z.uuid(), ok: z.literal(false), error: z.string().min(1).max(80) }).strict(),
+]);
+
+export class LocalNerUnavailableError extends Error {
+  constructor() {
+    super("local NER is unavailable");
+    this.name = "LocalNerUnavailableError";
+  }
+}
+
+interface PendingRequest {
+  resolve: (value: readonly LocalNerEvidence[]) => void;
+  reject: (error: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+  signal: AbortSignal;
+  cancel: () => void;
+}
+
+/** Persistent JSONL adapter for the optional, CPU-only Python NER worker. */
+export class PythonLocalNerDetector implements LocalNerDetector {
+  private child?: ChildProcessWithoutNullStreams;
+  private ready?: Promise<void>;
+  private readyResolve?: () => void;
+  private readyReject?: (error: Error) => void;
+  private workerReady = false;
+  private stdout = "";
+  private readonly pending = new Map<string, PendingRequest>();
+
+  constructor(private readonly options: {
+    pythonExecutable: string;
+    workerScript: string;
+    modelPath: string;
+    cwd: string;
+    threads?: number;
+    startupTimeoutMs?: number;
+  }) {}
+
+  async warmup(): Promise<void> {
+    await this.ensureStarted();
+  }
+
+  isReady(): boolean {
+    return this.workerReady
+      && this.child !== undefined
+      && this.child.exitCode === null
+      && this.child.signalCode === null;
+  }
+
+  async detect(
+    candidates: readonly LocalNerCandidate[],
+    signal: AbortSignal,
+    deadline: number,
+  ): Promise<readonly LocalNerEvidence[]> {
+    const parsed = z.array(candidateSchema).min(1).max(128).parse(candidates);
+    if (signal.aborted || deadline <= Date.now()) throw new LocalNerUnavailableError();
+    await this.ensureStartedWithin(signal, deadline);
+    if (!this.child || this.child.exitCode !== null || this.child.signalCode !== null) {
+      throw new LocalNerUnavailableError();
+    }
+    const id = randomUUID();
+    return await new Promise<readonly LocalNerEvidence[]>((resolve, reject) => {
+      const fail = () => {
+        this.finishPending(id);
+        reject(new LocalNerUnavailableError());
+        this.stopWorker();
+      };
+      const timer = setTimeout(fail, Math.max(1, Math.floor(deadline - Date.now())));
+      const cancel = fail;
+      const pending: PendingRequest = { resolve, reject, timer, signal, cancel };
+      this.pending.set(id, pending);
+      signal.addEventListener("abort", cancel, { once: true });
+      this.child!.stdin.write(`${JSON.stringify({ id, candidates: parsed })}\n`, (error) => {
+        if (error) fail();
+      });
+    });
+  }
+
+  async close(): Promise<void> {
+    const child = this.child;
+    if (!child || child.exitCode !== null || child.signalCode !== null) return;
+    await new Promise<void>((resolve) => {
+      child.once("close", () => resolve());
+      child.kill("SIGTERM");
+      setTimeout(() => {
+        if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
+      }, 250).unref();
+    });
+  }
+
+  private async ensureStarted(): Promise<void> {
+    if (this.ready) return await this.ready;
+    this.ready = new Promise<void>((resolve, reject) => {
+      this.readyResolve = resolve;
+      this.readyReject = reject;
+    });
+    const threads = String(this.options.threads ?? 2);
+    const inheritedRuntimeEnvironment = Object.fromEntries([
+      "PATH", "SystemRoot", "WINDIR", "PATHEXT", "TMPDIR", "TEMP", "TMP", "LANG", "LC_ALL",
+    ].flatMap((name) => process.env[name] === undefined ? [] : [[name, process.env[name]!]]));
+    const child = spawn(this.options.pythonExecutable, [
+      "-I",
+      "-B",
+      this.options.workerScript,
+      "--model",
+      this.options.modelPath,
+      "--threads",
+      threads,
+    ], {
+      cwd: this.options.cwd,
+      stdio: ["pipe", "pipe", "pipe"],
+      env: {
+        ...inheritedRuntimeEnvironment,
+        HOME: process.env.HOME ?? tmpdir(),
+        CUDA_VISIBLE_DEVICES: "",
+        HIP_VISIBLE_DEVICES: "",
+        HF_HUB_OFFLINE: "1",
+        HF_HUB_DISABLE_TELEMETRY: "1",
+        TRANSFORMERS_OFFLINE: "1",
+        TOKENIZERS_PARALLELISM: "false",
+        PYTHONNOUSERSITE: "1",
+        OMP_NUM_THREADS: threads,
+        MKL_NUM_THREADS: threads,
+        OPENBLAS_NUM_THREADS: threads,
+        HTTP_PROXY: "",
+        HTTPS_PROXY: "",
+        ALL_PROXY: "",
+        NO_PROXY: "*",
+      },
+    });
+    this.child = child;
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk: string) => this.receive(chunk));
+    child.stderr.resume();
+    child.once("error", () => this.failWorker());
+    child.once("close", () => this.failWorker());
+    const startupTimer = setTimeout(() => this.failWorker(), this.options.startupTimeoutMs ?? 120_000);
+    startupTimer.unref();
+    try {
+      await this.ready;
+    } finally {
+      clearTimeout(startupTimer);
+    }
+  }
+
+  private async ensureStartedWithin(signal: AbortSignal, deadline: number): Promise<void> {
+    const started = this.ensureStarted();
+    await new Promise<void>((resolve, reject) => {
+      let settled = false;
+      const finish = (error?: Error, stopWorker = false) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        signal.removeEventListener("abort", cancel);
+        if (stopWorker) this.failWorker();
+        if (error) reject(error);
+        else resolve();
+      };
+      const cancel = () => finish(new LocalNerUnavailableError(), true);
+      const timer = setTimeout(cancel, Math.max(1, Math.floor(deadline - Date.now())));
+      signal.addEventListener("abort", cancel, { once: true });
+      void started.then(
+        () => finish(),
+        () => finish(new LocalNerUnavailableError()),
+      );
+    });
+  }
+
+  private receive(chunk: string): void {
+    this.stdout += chunk;
+    if (Buffer.byteLength(this.stdout, "utf8") > MAX_LINE_BYTES) {
+      this.failWorker();
+      return;
+    }
+    let newline: number;
+    while ((newline = this.stdout.indexOf("\n")) >= 0) {
+      const line = this.stdout.slice(0, newline);
+      this.stdout = this.stdout.slice(newline + 1);
+      if (!line) continue;
+      try {
+        const message = workerMessageSchema.parse(JSON.parse(line));
+        if ("ready" in message) {
+          this.workerReady = true;
+          this.readyResolve?.();
+          this.readyResolve = undefined;
+          this.readyReject = undefined;
+          continue;
+        }
+        const pending = this.pending.get(message.id);
+        if (!pending) continue;
+        this.finishPending(message.id);
+        if (message.ok) pending.resolve(message.evidence);
+        else pending.reject(new LocalNerUnavailableError());
+      } catch {
+        this.failWorker();
+        return;
+      }
+    }
+  }
+
+  private finishPending(id: string): void {
+    const pending = this.pending.get(id);
+    if (!pending) return;
+    clearTimeout(pending.timer);
+    pending.signal.removeEventListener("abort", pending.cancel);
+    this.pending.delete(id);
+  }
+
+  private stopWorker(): void {
+    const child = this.child;
+    if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGTERM");
+  }
+
+  private failWorker(): void {
+    const error = new LocalNerUnavailableError();
+    this.readyReject?.(error);
+    this.readyResolve = undefined;
+    this.readyReject = undefined;
+    for (const [id, pending] of this.pending) {
+      this.finishPending(id);
+      pending.reject(error);
+    }
+    this.stopWorker();
+    this.child = undefined;
+    this.ready = undefined;
+    this.workerReady = false;
+    this.stdout = "";
+  }
+}

backend/src/catalog/memory-cleanup.ts

@@ -0,0 +1,23 @@
+import type { ThtRunner } from "../tht/tht-runner.js";
+import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
+import type { CatalogSyncRun, WorkspaceDatabase } from "./types.js";
+
+/** Internal continuation of an applied physical sync, using the harness Memory boundary. */
+export function createMemoryCleanup(runner: Pick<ThtRunner, "withPrincipal">, runtime: SemanticRuntimeConfig) {
+  return async (database: WorkspaceDatabase, run: CatalogSyncRun): Promise<number> => {
+    if (run.phase !== "memory_cleanup" || !run.plannedDiff) throw new Error("Physical cleanup is not committed");
+    const result = await runner.withPrincipal({ issuer: "installation", subject: "catalog-sync",
+      roles: ["admin"], permissions: ["memory.manage"], isAdmin: true,
+    }).runWithRuntimeSnapshot(["memory", "admin", "--workspace", database.workspaceId], JSON.stringify({
+      action: "cleanup", runtime, request: { sync_id: run.id, database: database.databaseName,
+        schema_name: database.schema, removed_tables: run.plannedDiff.deletedTables,
+        removed_columns: run.plannedDiff.deletedColumns.map(column => ({ table: column.tableName, column: column.columnName })),
+      },
+    }));
+    const payload = JSON.parse(result.stdout);
+    if (result.code !== 0 || payload.indexed !== true || !Number.isInteger(payload.deleted) || payload.deleted < 0) {
+      throw new Error("Memory cleanup is incomplete");
+    }
+    return payload.deleted;
+  };
+}

backend/src/catalog/memory-repository.ts

@@ -18,6 +18,8 @@ import {
   type CatalogLogicalRelationshipCandidate,
   type CatalogLogicalRelationshipContext,
   type CatalogPhysicalRelationship,
+  type CatalogPreprocessingStartResult,
+  type CatalogPreprocessingClearResult,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -35,10 +37,10 @@ import {
   type DescriptionGenerationRun,
   type DescriptionGenerationRunUpdate,
   type DescriptionGenerationScope,
-  type SensitiveDataSuggestionEvent,
-  type SensitiveDataSuggestionRun,
-  type SensitiveDataSuggestionRunUpdate,
-  type SensitiveDataSuggestionScope,
+  type SensitivityAnalysisEvent,
+  type SensitivityAnalysisRun,
+  type SensitivityAnalysisRunUpdate,
+  type SensitivityAnalysisScope,
   type TableSyncRepositoryResult,
   type WorkspaceDatabase,
 } from "./types.js";
@@ -56,8 +58,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
   private readonly logicalRelationships = new Map<string, CatalogLogicalRelationship>();
   private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
   private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
-  private readonly sensitiveDataSuggestionRuns = new Map<string, SensitiveDataSuggestionRun>();
-  private readonly sensitiveDataSuggestionEvents = new Map<string, SensitiveDataSuggestionEvent[]>();
+  private readonly sensitivityAnalysisRuns = new Map<string, SensitivityAnalysisRun>();
+  private readonly sensitivityAnalysisEvents = new Map<string, SensitivityAnalysisEvent[]>();
   private readonly syncRuns = new Map<string, CatalogSyncRun>();
   private readonly syncEvents = new Map<string, CatalogSyncEvent[]>();
 
@@ -72,6 +74,77 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
     return value ? clone(value) : undefined;
   }
+  async beginPreprocessing(
+    workspaceId: string,
+    inputFingerprint: string,
+  ): Promise<CatalogPreprocessingStartResult> {
+    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
+    if (!database) return { kind: "not_found" };
+    if (database.preprocessingStatus === "running") return { kind: "already_running" };
+    if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
+    const catalogBusy = [...this.syncRuns.values()].some((run) =>
+      run.databaseId === database.id
+      && ["queued", "running", "awaiting_confirmation", "applying"].includes(run.state))
+      || [...this.descriptionGenerationRuns.values()].some((run) =>
+        run.databaseId === database.id && ["queued", "running"].includes(run.status))
+      || [...this.sensitivityAnalysisRuns.values()].some((run) =>
+        run.databaseId === database.id && ["queued", "running"].includes(run.status));
+    if (catalogBusy) return { kind: "catalog_busy" };
+    const now = new Date().toISOString();
+    const updated: WorkspaceDatabase = {
+      ...database,
+      preprocessingStatus: "running",
+      preprocessingInputFingerprint: inputFingerprint,
+      preprocessedMetadataRevision: undefined,
+      preprocessingStartedAt: now,
+      preprocessingFinishedAt: undefined,
+      preprocessingErrorCode: undefined,
+      updatedAt: now,
+    };
+    this.records.set(database.id, updated);
+    return { kind: "started", database: clone(updated) };
+  }
+  async finishPreprocessing(
+    workspaceId: string,
+    metadataContentRevision: number,
+    inputFingerprint: string,
+    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
+  ): Promise<WorkspaceDatabase | undefined> {
+    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
+    if (!database
+      || database.preprocessingStatus !== "running"
+      || database.metadataContentRevision !== metadataContentRevision
+      || database.preprocessingInputFingerprint !== inputFingerprint) return undefined;
+    const updated: WorkspaceDatabase = {
+      ...database,
+      preprocessingStatus: outcome.status,
+      preprocessedMetadataRevision: outcome.status === "succeeded"
+        ? metadataContentRevision
+        : undefined,
+      preprocessingFinishedAt: new Date().toISOString(),
+      preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : undefined,
+    };
+    this.records.set(database.id, updated);
+    return clone(updated);
+  }
+  async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
+    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
+    if (!database) return { kind: "not_found" };
+    if (database.preprocessingStatus === "running") return { kind: "already_running" };
+    const now = new Date().toISOString();
+    const updated: WorkspaceDatabase = {
+      ...database,
+      preprocessingStatus: "failed",
+      preprocessingInputFingerprint: undefined,
+      preprocessedMetadataRevision: undefined,
+      preprocessingStartedAt: undefined,
+      preprocessingFinishedAt: now,
+      preprocessingErrorCode: "derived_data_cleared",
+      updatedAt: now,
+    };
+    this.records.set(database.id, updated);
+    return { kind: "cleared", database: clone(updated) };
+  }
   async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
     if (databaseId !== undefined && !this.records.has(databaseId)) return undefined;
 
@@ -83,8 +156,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const tableIds = new Set(tables.map((table) => table.id));
     const columns = [...this.columns.values()]
       .filter((column) => tableIds.has(column.tableId));
-    const relationships = [...this.relationships.values()]
-      .filter((relationship) => selectedDatabaseIds.has(relationship.databaseId));
+    const relationships = [
+      ...this.relationships.values(),
+      ...this.logicalRelationships.values(),
+    ].filter((relationship) => selectedDatabaseIds.has(relationship.databaseId));
 
     return createCatalogMetrics(databaseId, {
       tables: tables.length,
@@ -112,6 +187,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
       createdAt: now,
       updatedAt: now,
       connectionStatus: "untested",
+      metadataContentRevision: 0,
+      preprocessingStatus: "failed",
     };
     this.records.set(record.id, record);
     return clone(record);
@@ -183,10 +260,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
       this.descriptionGenerationRuns.delete(runId);
       this.descriptionGenerationEvents.delete(runId);
     }
-    for (const [runId, run] of this.sensitiveDataSuggestionRuns) {
+    for (const [runId, run] of this.sensitivityAnalysisRuns) {
       if (run.databaseId !== id) continue;
-      this.sensitiveDataSuggestionRuns.delete(runId);
-      this.sensitiveDataSuggestionEvents.delete(runId);
+      this.sensitivityAnalysisRuns.delete(runId);
+      this.sensitivityAnalysisEvents.delete(runId);
     }
     return this.records.delete(id);
   }
@@ -255,6 +332,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
     description: string | null,
     generatedDescription: string | null,
     sensitive?: boolean,
+    sensitivityReason?: string | null,
   ): Promise<CatalogColumn | undefined> {
     const current = await this.getColumn(databaseId, tableId, columnId);
     if (!current || current.version !== expectedVersion) return undefined;
@@ -263,6 +341,9 @@ export class MemoryCatalogRepository implements CatalogRepository {
       description,
       generatedDescription,
       sensitive: sensitive ?? current.sensitive,
+      sensitivityReason: sensitive === false
+        ? null
+        : sensitivityReason === undefined ? current.sensitivityReason : sensitivityReason,
       version: current.version + 1,
       updatedAt: new Date().toISOString(),
     };
@@ -276,10 +357,39 @@ export class MemoryCatalogRepository implements CatalogRepository {
     targetIds: readonly string[],
   ): Promise<CatalogDescriptionConsolidationCounts | undefined> {
     const selectedTargetIds = [...new Set(targetIds)];
-    if (!this.records.has(databaseId) || selectedTargetIds.length === 0) {
+    if (!this.records.has(databaseId)) {
       return undefined;
     }
     const now = new Date().toISOString();
+    if (target === "database" || target === "database_columns") {
+      const tableTargets = target === "database"
+        ? [...this.tables.values()].filter((table) => table.databaseId === databaseId)
+        : [];
+      const columnTargets = [...this.columns.values()].filter((column) => (
+        this.tables.get(column.tableId)?.databaseId === databaseId
+      ));
+      const copiedTables = tableTargets.filter((table) => Boolean(table.generatedDescription?.trim()));
+      const copiedColumns = columnTargets.filter((column) => Boolean(column.generatedDescription?.trim()));
+      for (const table of copiedTables) {
+        this.tables.set(table.id, {
+          ...table,
+          description: table.generatedDescription,
+          version: table.version + 1,
+          updatedAt: now,
+        });
+      }
+      for (const column of copiedColumns) {
+        this.columns.set(column.id, {
+          ...column,
+          description: column.generatedDescription,
+          version: column.version + 1,
+          updatedAt: now,
+        });
+      }
+      const copied = copiedTables.length + copiedColumns.length;
+      return { copied, skipped: tableTargets.length + columnTargets.length - copied };
+    }
+    if (selectedTargetIds.length === 0) return undefined;
     if (target === "tables") {
       const targets = selectedTargetIds.map((id) => this.tables.get(id));
       if (targets.some((table) => !table || table.databaseId !== databaseId)) return undefined;
@@ -433,93 +543,96 @@ export class MemoryCatalogRepository implements CatalogRepository {
       .map((event) => structuredClone(event));
   }
 
-  async createSensitiveDataSuggestionRun(
+  async createSensitivityAnalysisRun(
     databaseId: string,
-    scope: SensitiveDataSuggestionScope,
-    modelId: string,
-  ): Promise<SensitiveDataSuggestionRun> {
+    scope: SensitivityAnalysisScope,
+    origin: { engine: "llm"; modelId: string } | { engine: "local"; policyVersion: string },
+  ): Promise<SensitivityAnalysisRun> {
     const now = new Date().toISOString();
-    const run: SensitiveDataSuggestionRun = {
+    const run: SensitivityAnalysisRun = {
       id: randomUUID(),
       databaseId,
       scope,
-      modelId,
+      engine: origin.engine,
+      modelId: origin.engine === "llm" ? origin.modelId : null,
+      policyVersion: origin.engine === "local" ? origin.policyVersion : null,
       status: "running",
       total: 0,
       suggestedSensitive: 0,
-        suggestedNonSensitive: 0,
-        inputTokens: 0,
-        cacheReadTokens: 0,
-        outputTokens: 0,
+      suggestedNonSensitive: 0,
+      unknown: 0,
+      inputTokens: 0,
+      cacheReadTokens: 0,
+      outputTokens: 0,
       createdAt: now,
       startedAt: now,
       updatedAt: now,
       finishedAt: null,
       errorSummary: null,
     };
-    this.sensitiveDataSuggestionRuns.set(run.id, run);
+    this.sensitivityAnalysisRuns.set(run.id, run);
     return structuredClone(run);
   }
 
-  async getSensitiveDataSuggestionRun(
+  async getSensitivityAnalysisRun(
     runId: string,
-  ): Promise<SensitiveDataSuggestionRun | undefined> {
-    const run = this.sensitiveDataSuggestionRuns.get(runId);
+  ): Promise<SensitivityAnalysisRun | undefined> {
+    const run = this.sensitivityAnalysisRuns.get(runId);
     return run ? structuredClone(run) : undefined;
   }
 
-  async listSensitiveDataSuggestionRuns(limit = 50): Promise<SensitiveDataSuggestionRun[]> {
-    return [...this.sensitiveDataSuggestionRuns.values()]
+  async listSensitivityAnalysisRuns(limit = 50): Promise<SensitivityAnalysisRun[]> {
+    return [...this.sensitivityAnalysisRuns.values()]
       .sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id))
       .slice(0, limit)
       .map((run) => structuredClone(run));
   }
 
-  async interruptActiveSensitiveDataSuggestionRuns(
+  async interruptActiveSensitivityAnalysisRuns(
     errorSummary: string,
-  ): Promise<SensitiveDataSuggestionRun[]> {
-    const interrupted: SensitiveDataSuggestionRun[] = [];
-    for (const run of this.sensitiveDataSuggestionRuns.values()) {
+  ): Promise<SensitivityAnalysisRun[]> {
+    const interrupted: SensitivityAnalysisRun[] = [];
+    for (const run of this.sensitivityAnalysisRuns.values()) {
       if (run.status !== "running") continue;
       const now = new Date().toISOString();
-      const updated: SensitiveDataSuggestionRun = {
+      const updated: SensitivityAnalysisRun = {
         ...run,
         status: "interrupted",
         updatedAt: now,
         finishedAt: now,
         errorSummary,
       };
-      this.sensitiveDataSuggestionRuns.set(run.id, updated);
+      this.sensitivityAnalysisRuns.set(run.id, updated);
       interrupted.push(structuredClone(updated));
     }
     return interrupted;
   }
 
-  async updateSensitiveDataSuggestionRun(
+  async updateSensitivityAnalysisRun(
     runId: string,
-    update: SensitiveDataSuggestionRunUpdate,
-  ): Promise<SensitiveDataSuggestionRun | undefined> {
-    const current = this.sensitiveDataSuggestionRuns.get(runId);
+    update: SensitivityAnalysisRunUpdate,
+  ): Promise<SensitivityAnalysisRun | undefined> {
+    const current = this.sensitivityAnalysisRuns.get(runId);
     if (!current) return undefined;
     const updated = {
       ...current,
       ...structuredClone(update),
       updatedAt: new Date().toISOString(),
     };
-    this.sensitiveDataSuggestionRuns.set(runId, updated);
+    this.sensitivityAnalysisRuns.set(runId, updated);
     return structuredClone(updated);
   }
 
-  async appendSensitiveDataSuggestionEvent(
+  async appendSensitivityAnalysisEvent(
     runId: string,
-    level: SensitiveDataSuggestionEvent["level"],
+    level: SensitivityAnalysisEvent["level"],
     message: string,
-  ): Promise<SensitiveDataSuggestionEvent> {
-    if (!this.sensitiveDataSuggestionRuns.has(runId)) {
-      throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist");
+  ): Promise<SensitivityAnalysisEvent> {
+    if (!this.sensitivityAnalysisRuns.has(runId)) {
+      throw new CatalogConflictError("Sensitivity Analysis Run does not exist");
     }
-    const events = this.sensitiveDataSuggestionEvents.get(runId) ?? [];
-    const event: SensitiveDataSuggestionEvent = {
+    const events = this.sensitivityAnalysisEvents.get(runId) ?? [];
+    const event: SensitivityAnalysisEvent = {
       runId,
       sequence: events.length + 1,
       level,
@@ -527,15 +640,15 @@ export class MemoryCatalogRepository implements CatalogRepository {
       createdAt: new Date().toISOString(),
     };
     events.push(event);
-    this.sensitiveDataSuggestionEvents.set(runId, events);
+    this.sensitivityAnalysisEvents.set(runId, events);
     return structuredClone(event);
   }
 
-  async listSensitiveDataSuggestionEvents(
+  async listSensitivityAnalysisEvents(
     runId: string,
     afterSequence = 0,
-  ): Promise<SensitiveDataSuggestionEvent[]> {
-    return (this.sensitiveDataSuggestionEvents.get(runId) ?? [])
+  ): Promise<SensitivityAnalysisEvent[]> {
+    return (this.sensitivityAnalysisEvents.get(runId) ?? [])
       .filter((event) => event.sequence > afterSequence)
       .map((event) => structuredClone(event));
   }
@@ -684,18 +797,22 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const tables = [...this.tables.values()].filter((table) => selected.has(table.databaseId));
     const tableIds = new Set(tables.map((table) => table.id));
     const columns = [...this.columns.values()].filter((column) => tableIds.has(column.tableId));
-    const relationships = [...this.relationships.values()]
+    const physicalRelationships = [...this.relationships.values()]
       .filter((relationship) => selected.has(relationship.databaseId));
+    const logicalRelationships = [...this.logicalRelationships.values()]
+      .filter((relationship) => selected.has(relationship.databaseId));
+    const relationshipCount = physicalRelationships.length + logicalRelationships.length;
 
     if (target === "tables") {
       for (const table of tables) this.deleteTable(table.id);
       this.markCatalogIncomplete(selectedDatabaseIds);
-      return { tables: tables.length, columns: columns.length, relationships: relationships.length };
+      return { tables: tables.length, columns: columns.length, relationships: relationshipCount };
     }
-    for (const relationship of relationships) this.relationships.delete(relationship.id);
+    for (const relationship of physicalRelationships) this.relationships.delete(relationship.id);
+    for (const relationship of logicalRelationships) this.logicalRelationships.delete(relationship.id);
     for (const databaseId of selectedDatabaseIds) this.refreshForeignKeyFlags(databaseId);
     this.markCatalogIncomplete(selectedDatabaseIds);
-    return { tables: 0, columns: 0, relationships: relationships.length };
+    return { tables: 0, columns: 0, relationships: relationshipCount };
   }
 
   async deleteTableMetadata(
@@ -733,14 +850,23 @@ export class MemoryCatalogRepository implements CatalogRepository {
       return { tables: 0, columns: columns.length, relationships: 0 };
     }
 
-    const relationships = [...this.relationships.values()].filter((relationship) => (
+    const physicalRelationships = [...this.relationships.values()].filter((relationship) => (
       relationship.databaseId === databaseId
       && (selected.has(relationship.sourceTableId) || selected.has(relationship.targetTableId))
     ));
-    for (const relationship of relationships) this.relationships.delete(relationship.id);
+    const logicalRelationships = [...this.logicalRelationships.values()].filter((relationship) => (
+      relationship.databaseId === databaseId
+      && (selected.has(relationship.sourceTableId) || selected.has(relationship.targetTableId))
+    ));
+    for (const relationship of physicalRelationships) this.relationships.delete(relationship.id);
+    for (const relationship of logicalRelationships) this.logicalRelationships.delete(relationship.id);
     this.refreshForeignKeyFlags(databaseId);
     this.markCatalogIncomplete([databaseId]);
-    return { tables: 0, columns: 0, relationships: relationships.length };
+    return {
+      tables: 0,
+      columns: 0,
+      relationships: physicalRelationships.length + logicalRelationships.length,
+    };
   }
 
   async planSchemaSync(
@@ -800,6 +926,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
     scope: CatalogSyncScope,
     tableIds: readonly string[],
     snapshot: ObservedSchemaSnapshot,
+    syncRunId?: string,
   ): Promise<CatalogSyncCounts | undefined> {
     const database = this.records.get(databaseId);
     if (!database || database.version !== expectedDatabaseVersion) return undefined;
@@ -924,6 +1051,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
             description: null,
             generatedDescription: null,
             sensitive: false,
+            sensitivityReason: null,
             lastSyncedDatabaseVersion: expectedDatabaseVersion,
             lastSyncedAt: now,
             version: 1,
@@ -1014,6 +1142,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
       if (scope === "all") {
         this.records.set(databaseId, { ...database, schemaSyncedVersion: expectedDatabaseVersion, schemaSyncedAt: now });
       }
+      if (syncRunId) await this.updateSyncRun(syncRunId, { phase: "memory_cleanup" });
       return {
         tables: (await this.listTables(databaseId)).length,
         columns: [...this.columns.values()].filter((column) => this.tables.get(column.tableId)?.databaseId === databaseId).length,
@@ -1127,7 +1256,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
     for (const run of this.syncRuns.values()) {
       if (["queued", "running", "awaiting_confirmation", "applying"].includes(run.state)) {
         await this.updateSyncRun(run.id, {
-          state: "interrupted", phase: "completed", finishedAt: new Date().toISOString(),
+          state: "interrupted", phase: run.phase === "memory_cleanup" ? "memory_cleanup" : "completed", finishedAt: new Date().toISOString(),
           errorCode: "SYNC_INTERRUPTED", errorMessage: "Synchronization was interrupted by a service restart",
         });
       }

backend/src/catalog/metadata-generation-models.ts

@@ -1,63 +1,5 @@
-import {
-  closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
-  type Stats,
-} from "node:fs";
-import { parseAllDocuments } from "yaml";
-import { z } from "zod";
-import {
-  loadSecretBundle,
-  METADATA_GENERATION_SECRET_KEYS,
-} from "../config/secret-bundle.js";
-
-const MAX_INSTALLATION_BYTES = 1024 * 1024;
-const RUNTIME_INSTALLATION_FILE = "/run/thothii-installation/thothii-installation.yaml";
-const modelId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
-const apiKeyEnvironment = z.enum(METADATA_GENERATION_SECRET_KEYS);
-const endpointSchema = z.object({
-  baseUrl: z.string().min(1).max(2048).refine((value) => {
-    try {
-      const url = new URL(value);
-      return (url.protocol === "http:" || url.protocol === "https:")
-        && url.username === "" && url.password === "" && url.search === "" && url.hash === "";
-    } catch {
-      return false;
-    }
-  }),
-  apiVersion: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/).optional(),
-}).strict();
-const configuredModelSchema = z.object({
-  id: modelId,
-  label: z.string().min(1).max(128).refine((value) => value.trim() === value && !/\p{Cc}/u.test(value)),
-  litellm: z.object({
-    provider: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/),
-    model: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/),
-    disableThinking: z.literal(true).optional(),
-    endpoint: endpointSchema.optional(),
-  }).strict(),
-  apiKeyEnv: apiKeyEnvironment.optional(),
-}).strict().superRefine((value, context) => {
-  if (value.apiKeyEnv === undefined && value.litellm.endpoint === undefined) {
-    context.addIssue({
-      code: z.ZodIssueCode.custom,
-      path: ["apiKeyEnv"],
-      message: "keyless models require an explicit endpoint",
-    });
-  }
-  if (value.litellm.disableThinking === true && value.litellm.endpoint === undefined) {
-    context.addIssue({
-      code: z.ZodIssueCode.custom,
-      path: ["litellm", "disableThinking"],
-      message: "thinking may be disabled only for an explicit endpoint",
-    });
-  }
-});
-const metadataGenerationSchema = z.object({
-  default: modelId.optional(),
-  models: z.array(configuredModelSchema).max(64).default([]),
-}).strict();
-const installationSchema = z.object({
-  metadataGeneration: metadataGenerationSchema.optional(),
-}).passthrough();
+import { loadSecretBundle } from "../config/secret-bundle.js";
+import { loadRuntimeModelCatalog } from "../models/runtime-model-catalog.js";
 
 export interface MetadataGenerationModelChoice {
   id: string;
@@ -86,7 +28,6 @@ export class MetadataGenerationModelUnavailableError extends Error {
   }
 }
 
-/** The complete interface callers need: safe discovery plus fail-closed runtime resolution. */
 export interface MetadataGenerationModels {
   catalog(): MetadataGenerationModelCatalog;
   resolve(selection: string): ResolvedMetadataGenerationModel;
@@ -96,140 +37,78 @@ class RestartLoadedMetadataGenerationModels implements MetadataGenerationModels
   readonly #models: ReadonlyMap<string, ResolvedMetadataGenerationModel>;
   readonly #catalog: MetadataGenerationModelCatalog;
 
-  constructor(
-    models: ReadonlyMap<string, ResolvedMetadataGenerationModel> = new Map(),
-    defaultModel: string | null = null,
-    choices: MetadataGenerationModelChoice[] = [],
-  ) {
+  constructor(models: ReadonlyMap<string, ResolvedMetadataGenerationModel>, defaultModel: string | null) {
     this.#models = models;
     this.#catalog = {
-      models: choices.map((choice) => ({ ...choice })),
+      models: [...models.values()].map(({ id }) => ({ id, label: id })),
       default: defaultModel,
     };
   }
 
   catalog(): MetadataGenerationModelCatalog {
-    return {
-      models: this.#catalog.models.map((choice) => ({ ...choice })),
-      default: this.#catalog.default,
-    };
+    return { models: this.#catalog.models.map((choice) => ({ ...choice })), default: this.#catalog.default };
   }
 
   resolve(selection: string): ResolvedMetadataGenerationModel {
-    const model = typeof selection === "string" ? this.#models.get(selection) : undefined;
+    const model = this.#models.get(selection);
     if (!model) throw new MetadataGenerationModelUnavailableError();
     return model;
   }
 }
 
-function invalid(message = "metadata-generation configuration is invalid"): Error {
+function invalid(message = "metadata-generation runtime catalog is invalid"): Error {
   return new Error(message);
 }
 
-function protectedInstallationStat(file: string, info: Stats): boolean {
-  const mode = info.mode & 0o777;
-  if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
-    || info.size < 1 || info.size > MAX_INSTALLATION_BYTES) return false;
-  if (file === RUNTIME_INSTALLATION_FILE && info.uid === 0 && mode === 0o444) return true;
-  return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600);
-}
-
-function readProtectedInstallation(file: string): string {
-  let descriptor: number | undefined;
-  try {
-    const before = lstatSync(file);
-    if (!protectedInstallationStat(file, before)) throw new Error("unavailable");
-    descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
-    const opened = fstatSync(descriptor);
-    if (!protectedInstallationStat(file, opened)
-      || before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("unavailable");
-    const source = readFileSync(descriptor, "utf8");
-    const after = fstatSync(descriptor);
-    const current = lstatSync(file);
-    if (!protectedInstallationStat(file, after) || !protectedInstallationStat(file, current)
-      || opened.dev !== after.dev || opened.ino !== after.ino
-      || opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("unavailable");
-    return source;
-  } finally {
-    if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized below */ }
-  }
-}
-
-function readInstallation(file: string): unknown {
-  try {
-    const documents = parseAllDocuments(readProtectedInstallation(file), { uniqueKeys: true });
-    if (documents.length !== 1) throw invalid("metadata-generation installation must contain one YAML document");
-    const document = documents[0];
-    if (document.errors.length > 0 || document.warnings.length > 0) {
-      throw invalid("metadata-generation installation contains invalid YAML");
-    }
-    return document.toJSON();
-  } catch (error) {
-    if (error instanceof Error && error.message.startsWith("metadata-generation")) throw error;
-    throw invalid("metadata-generation installation is unavailable");
-  }
-}
-
 export function loadMetadataGenerationModels(options: {
-  installationFile?: string;
+  catalogFile?: string;
   secretsFile?: string;
 }): MetadataGenerationModels {
-  if (!options.installationFile) return new RestartLoadedMetadataGenerationModels();
-  const installation = installationSchema.safeParse(readInstallation(options.installationFile));
-  if (!installation.success) throw invalid();
-  const configured = installation.data.metadataGeneration;
-  if (!configured || configured.models.length === 0) {
-    if (configured?.default !== undefined) throw invalid("metadata-generation default does not identify a configured model");
-    return new RestartLoadedMetadataGenerationModels();
-  }
-  if (!configured.default) throw invalid("metadata-generation default is required when models are configured");
+  const catalog = loadRuntimeModelCatalog(options.catalogFile);
+  const configured = catalog.metadataModels();
+  if (configured.length === 0) return new RestartLoadedMetadataGenerationModels(new Map(), null);
 
-  const seen = new Set<string>();
-  for (const model of configured.models) {
-    if (seen.has(model.id)) throw invalid(`metadata-generation model id "${model.id}" is duplicated`);
-    seen.add(model.id);
-  }
-  if (!seen.has(configured.default)) {
-    throw invalid(`metadata-generation default "${configured.default}" is not configured`);
-  }
-  const requiresSecrets = configured.models.some((model) => model.apiKeyEnv !== undefined);
+  const requiresSecrets = configured.some((model) => model.authentication.mode === "secret_env");
   let secrets: ReadonlyMap<string, string> = new Map();
   if (requiresSecrets) {
     if (!options.secretsFile) throw invalid("metadata-generation keyed models require THT_SECRETS_FILE");
-    try {
-      secrets = loadSecretBundle(options.secretsFile);
-    } catch {
-      throw invalid("metadata-generation secrets are unavailable");
-    }
+    try { secrets = loadSecretBundle(options.secretsFile); }
+    catch { throw invalid("metadata-generation secrets are unavailable"); }
   }
+
   const models = new Map<string, ResolvedMetadataGenerationModel>();
-  for (const configuredModel of configured.models) {
+  const labels = new Map<string, string>();
+  for (const configuredModel of configured) {
+    const adapter = configuredModel.metadataAdapter;
+    if (!adapter || configuredModel.authentication.mode === "pi_auth") throw invalid();
+    const apiKeyEnv = configuredModel.authentication.apiKeyEnv;
     let apiKey: string | undefined;
-    if (configuredModel.apiKeyEnv !== undefined) {
-      apiKey = secrets.get(configuredModel.apiKeyEnv);
-      if (!apiKey) {
-        throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is missing`);
-      }
+    if (configuredModel.authentication.mode === "secret_env") {
+      if (!apiKeyEnv) throw invalid();
+      apiKey = secrets.get(apiKeyEnv);
+      if (!apiKey) throw invalid(`metadata-generation model "${configuredModel.id}" secret "${apiKeyEnv}" is missing`);
       if (apiKey.length > 16 * 1024 || /\s/u.test(apiKey)) {
-        throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is unusable`);
+        throw invalid(`metadata-generation model "${configuredModel.id}" secret "${apiKeyEnv}" is unusable`);
       }
     }
+    labels.set(configuredModel.id, configuredModel.label);
     models.set(configuredModel.id, Object.freeze({
       id: configuredModel.id,
-      provider: configuredModel.litellm.provider,
-      model: configuredModel.litellm.model,
-      ...(configuredModel.litellm.disableThinking === true ? { disableThinking: true as const } : {}),
-      ...(configuredModel.litellm.endpoint === undefined
-        ? {}
-        : { endpoint: Object.freeze({ ...configuredModel.litellm.endpoint }) }),
-      ...(configuredModel.apiKeyEnv === undefined
-        ? {}
-        : { apiKeyEnv: configuredModel.apiKeyEnv, apiKey }),
+      provider: adapter.litellmProvider,
+      model: configuredModel.upstreamModel,
+      ...(configuredModel.metadataGeneration?.disableThinking === true
+        ? { disableThinking: true as const } : {}),
+      ...(configuredModel.endpoint ? { endpoint: Object.freeze({ ...configuredModel.endpoint }) } : {}),
+      ...(apiKeyEnv ? { apiKeyEnv, apiKey } : {}),
     }));
   }
-  return new RestartLoadedMetadataGenerationModels(
-    models,
-    configured.default,
-    configured.models.map(({ id, label }) => ({ id, label })),
-  );
+  const result = new RestartLoadedMetadataGenerationModels(models, models.size ? catalog.defaultInteraction : null);
+  const safe = result.catalog();
+  return {
+    catalog: () => ({
+      default: safe.default,
+      models: safe.models.map((choice) => ({ ...choice, label: labels.get(choice.id) ?? choice.id })),
+    }),
+    resolve: (selection) => result.resolve(selection),
+  };
 }

backend/src/catalog/metadata-snapshot.ts

@@ -0,0 +1,144 @@
+import type {
+  CatalogColumn,
+  CatalogLogicalRelationship,
+  CatalogPhysicalRelationship,
+  CatalogRepository,
+  CatalogTable,
+} from "./types.js";
+
+export type CatalogDescriptionSource = "curated" | "generated" | "source_comment";
+
+export interface CatalogMetadataSnapshotColumn {
+  id: string;
+  name: string;
+  ordinalPosition: number;
+  dataType: string;
+  isNullable: boolean;
+  defaultExpression: string | null;
+  primaryKeyPosition: number | null;
+  sensitive: boolean;
+  description: string | null;
+  descriptionSource: CatalogDescriptionSource | null;
+}
+
+export interface CatalogMetadataSnapshotTable {
+  id: string;
+  name: string;
+  description: string | null;
+  descriptionSource: CatalogDescriptionSource | null;
+  columns: CatalogMetadataSnapshotColumn[];
+}
+
+export interface CatalogMetadataSnapshotRelationship {
+  id: string;
+  origin: "physical" | "generated" | "manual";
+  sourceTable: string;
+  sourceColumns: string[];
+  targetTable: string;
+  targetColumns: string[];
+}
+
+export interface CatalogMetadataSnapshot {
+  schemaVersion: 1;
+  workspaceId: string;
+  databaseId: string;
+  databaseName: string;
+  schemaName: string;
+  metadataContentRevision: number;
+  tables: CatalogMetadataSnapshotTable[];
+  relationships: CatalogMetadataSnapshotRelationship[];
+}
+
+function effectiveDescription(value: {
+  description: string | null;
+  generatedDescription: string | null;
+  sourceComment: string | null;
+}): { description: string | null; descriptionSource: CatalogDescriptionSource | null } {
+  if (value.description?.trim()) {
+    return { description: value.description.trim(), descriptionSource: "curated" };
+  }
+  if (value.generatedDescription?.trim()) {
+    return { description: value.generatedDescription.trim(), descriptionSource: "generated" };
+  }
+  if (value.sourceComment?.trim()) {
+    return { description: value.sourceComment.trim(), descriptionSource: "source_comment" };
+  }
+  return { description: null, descriptionSource: null };
+}
+
+function snapshotColumn(column: CatalogColumn): CatalogMetadataSnapshotColumn {
+  return {
+    id: column.id,
+    name: column.name,
+    ordinalPosition: column.ordinalPosition,
+    dataType: column.dataType,
+    isNullable: column.isNullable,
+    defaultExpression: column.defaultExpression,
+    primaryKeyPosition: column.primaryKeyPosition,
+    sensitive: column.sensitive,
+    ...effectiveDescription(column),
+  };
+}
+
+function snapshotRelationship(
+  relationship: CatalogPhysicalRelationship | CatalogLogicalRelationship,
+): CatalogMetadataSnapshotRelationship {
+  const columns = [...relationship.columns].sort((left, right) => left.position - right.position);
+  return {
+    id: relationship.id,
+    origin: relationship.origin,
+    sourceTable: relationship.sourceTableName,
+    sourceColumns: columns.map((column) => column.sourceColumnName),
+    targetTable: relationship.targetTableName,
+    targetColumns: columns.map((column) => column.targetColumnName),
+  };
+}
+
+export async function buildCatalogMetadataSnapshot(
+  repository: CatalogRepository,
+  workspaceId: string,
+  expectedMetadataContentRevision: number,
+): Promise<CatalogMetadataSnapshot> {
+  const database = await repository.getByWorkspace(workspaceId);
+  if (!database || database.preprocessingStatus !== "running") {
+    throw new Error("catalog preprocessing lease is not active");
+  }
+  if (database.metadataContentRevision !== expectedMetadataContentRevision) {
+    throw new Error("catalog metadata revision changed");
+  }
+
+  const catalogTables = await repository.listTables(database.id);
+  const tables: CatalogMetadataSnapshotTable[] = [];
+  for (const table of [...catalogTables].sort((left, right) => left.name.localeCompare(right.name))) {
+    const columns = await repository.listColumns(database.id, table.id);
+    tables.push({
+      id: table.id,
+      name: table.name,
+      ...effectiveDescription(table),
+      columns: columns
+        .sort((left, right) => left.ordinalPosition - right.ordinalPosition || left.name.localeCompare(right.name))
+        .map(snapshotColumn),
+    });
+  }
+
+  const physical = await repository.listRelationships(database.id);
+  const logical = (await repository.listLogicalRelationships(database.id))
+    .filter((relationship) => relationship.status === "active");
+  const relationships = [...physical, ...logical]
+    .map(snapshotRelationship)
+    .sort((left, right) =>
+      left.sourceTable.localeCompare(right.sourceTable)
+      || left.targetTable.localeCompare(right.targetTable)
+      || left.id.localeCompare(right.id));
+
+  return {
+    schemaVersion: 1,
+    workspaceId,
+    databaseId: database.id,
+    databaseName: database.databaseName,
+    schemaName: database.schema,
+    metadataContentRevision: expectedMetadataContentRevision,
+    tables,
+    relationships,
+  };
+}

backend/src/catalog/migrate.ts

@@ -9,9 +9,13 @@ import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_syn
 import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js";
 import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
 import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
-import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
+import * as sensitivityAnalysisRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
 import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
 import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
+import * as canonicalModelIdsMigration from "./migrations/010_canonical_model_ids.js";
+import * as localSensitivityAnalysisMigration from "./migrations/011_local_sensitivity_analysis.js";
+import * as sensitivityReasonMigration from "./migrations/012_sensitivity_reason.js";
+import * as catalogPreprocessingStateMigration from "./migrations/013_catalog_preprocessing_state.js";
 
 const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
 const host = process.env.THT_CATALOG_DB_HOST;
@@ -43,9 +47,13 @@ const provider: MigrationProvider = {
       "004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration,
       "005_description_generation_runs": descriptionGenerationRunsMigration,
       "006_sensitive_data_flag": sensitiveDataFlagMigration,
-      "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
+      "007_sensitive_data_suggestion_runs": sensitivityAnalysisRunsMigration,
       "008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
       "009_ai_token_usage": aiTokenUsageMigration,
+      "010_canonical_model_ids": canonicalModelIdsMigration,
+      "011_local_sensitivity_analysis": localSensitivityAnalysisMigration,
+      "012_sensitivity_reason": sensitivityReasonMigration,
+      "013_catalog_preprocessing_state": catalogPreprocessingStateMigration,
     };
   },
 };

backend/src/catalog/migrations/010_canonical_model_ids.ts

@@ -0,0 +1,28 @@
+import { sql, type Kysely } from "kysely";
+import type { CatalogDatabase } from "../repository.js";
+
+const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$";
+const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$";
+const historicalOrCanonicalModelPattern = `(${legacyModelPattern})|(${canonicalModelPattern})`;
+
+export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
+  await sql.raw(`alter table description_generation_runs
+    drop constraint description_generation_runs_model_id_check,
+    add constraint description_generation_runs_model_id_check
+      check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
+  await sql.raw(`alter table sensitive_data_suggestion_runs
+    drop constraint sensitive_data_suggestion_runs_model_id_check,
+    add constraint sensitive_data_suggestion_runs_model_id_check
+      check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
+}
+
+export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+  await sql.raw(`alter table sensitive_data_suggestion_runs
+    drop constraint sensitive_data_suggestion_runs_model_id_check,
+    add constraint sensitive_data_suggestion_runs_model_id_check
+      check (model_id ~ '${legacyModelPattern}')`).execute(db);
+  await sql.raw(`alter table description_generation_runs
+    drop constraint description_generation_runs_model_id_check,
+    add constraint description_generation_runs_model_id_check
+      check (model_id ~ '${legacyModelPattern}')`).execute(db);
+}

backend/src/catalog/migrations/011_local_sensitivity_analysis.ts

@@ -0,0 +1,42 @@
+import { sql, type Kysely } from "kysely";
+import type { CatalogDatabase } from "../repository.js";
+
+export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
+  await sql.raw(`alter table sensitive_data_suggestion_runs
+    alter column model_id drop not null,
+    add column engine text not null default 'llm',
+    add column policy_version text,
+    add column unknown integer not null default 0,
+    drop constraint sensitive_data_suggestion_runs_counters_check,
+    add constraint sensitive_data_suggestion_runs_counters_check
+      check (total >= 0
+        and suggested_sensitive >= 0
+        and suggested_non_sensitive >= 0
+        and unknown >= 0
+        and suggested_sensitive + suggested_non_sensitive + unknown <= total),
+    add constraint sensitive_data_suggestion_runs_engine_check
+      check (engine in ('llm', 'local')),
+    add constraint sensitive_data_suggestion_runs_origin_check
+      check ((engine = 'llm' and model_id is not null and policy_version is null)
+        or (engine = 'local' and model_id is null
+          and policy_version ~ '^[a-z][a-z0-9._-]{0,63}$'))`).execute(db);
+}
+
+export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+  await sql.raw(`alter table sensitive_data_suggestion_runs
+    drop constraint sensitive_data_suggestion_runs_origin_check,
+    drop constraint sensitive_data_suggestion_runs_engine_check,
+    drop constraint sensitive_data_suggestion_runs_counters_check`).execute(db);
+  await sql.raw(`update sensitive_data_suggestion_runs
+    set model_id = coalesce(model_id, 'local/sensitivity-v1')`).execute(db);
+  await sql.raw(`alter table sensitive_data_suggestion_runs
+    drop column unknown,
+    drop column policy_version,
+    drop column engine,
+    alter column model_id set not null,
+    add constraint sensitive_data_suggestion_runs_counters_check
+      check (total >= 0
+        and suggested_sensitive >= 0
+        and suggested_non_sensitive >= 0
+        and suggested_sensitive + suggested_non_sensitive <= total)`).execute(db);
+}

backend/src/catalog/migrations/012_sensitivity_reason.ts

@@ -0,0 +1,12 @@
+import type { Kysely } from "kysely";
+import type { CatalogDatabase } from "../repository.js";
+
+export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.alterTable("catalog_columns")
+    .addColumn("sensitivity_reason", "text")
+    .execute();
+}
+
+export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.alterTable("catalog_columns").dropColumn("sensitivity_reason").execute();
+}

backend/src/catalog/migrations/013_catalog_preprocessing_state.ts

@@ -0,0 +1,212 @@
+import { type Kysely, sql } from "kysely";
+import type { CatalogDatabase } from "../repository.js";
+
+export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.alterTable("workspace_databases")
+    .addColumn("metadata_content_revision", "bigint", (column) => column.notNull().defaultTo(0))
+    .addColumn("preprocessing_status", "text", (column) => column.notNull().defaultTo("failed"))
+    .addColumn("preprocessing_input_fingerprint", "text")
+    .addColumn("preprocessed_metadata_revision", "bigint")
+    .addColumn("preprocessing_started_at", "timestamptz")
+    .addColumn("preprocessing_finished_at", "timestamptz")
+    .addColumn("preprocessing_error_code", "text")
+    .execute();
+  await sql`
+    alter table workspace_databases
+    add constraint workspace_databases_preprocessing_status_check
+    check (preprocessing_status in ('running', 'succeeded', 'failed'))
+  `.execute(db);
+
+  await sql`
+    create function catalog_metadata_write_guard()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      resolved_database_id uuid;
+      current_status text;
+      relation_id uuid;
+      table_id uuid;
+    begin
+      if tg_table_name = 'catalog_tables' then
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      elsif tg_table_name = 'catalog_columns' then
+        table_id := coalesce(new.table_id, old.table_id);
+        select database_id into resolved_database_id from catalog_tables where id = table_id;
+      elsif tg_table_name = 'catalog_relationships' then
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      elsif tg_table_name = 'catalog_relationship_columns' then
+        relation_id := coalesce(new.relationship_id, old.relationship_id);
+        select database_id into resolved_database_id from catalog_relationships where id = relation_id;
+      elsif tg_table_name = 'catalog_logical_relationships' then
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      elsif tg_table_name = 'database_bindings' then
+        if tg_op = 'UPDATE' and not (
+          new.transport is distinct from old.transport
+          or new.host is distinct from old.host
+          or new.port is distinct from old.port
+          or new.username is distinct from old.username
+          or new.base_url is distinct from old.base_url
+          or new.rest_path is distinct from old.rest_path
+          or new.rest_auth is distinct from old.rest_auth
+          or new.tls_servername is distinct from old.tls_servername
+          or new.ssh_host is distinct from old.ssh_host
+          or new.ssh_port is distinct from old.ssh_port
+          or new.ssh_username is distinct from old.ssh_username
+          or new.ssh_target_host is distinct from old.ssh_target_host
+          or new.ssh_target_port is distinct from old.ssh_target_port
+        ) then
+          return new;
+        end if;
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      end if;
+      if resolved_database_id is null then
+        if tg_op = 'DELETE' then return old; else return new; end if;
+      end if;
+
+      select preprocessing_status into current_status
+      from workspace_databases
+      where id = resolved_database_id
+      for update;
+
+      if current_status = 'running' then
+        raise exception 'catalog preprocessing is running'
+          using errcode = '55000';
+      end if;
+
+      update workspace_databases
+      set metadata_content_revision = metadata_content_revision + 1,
+          preprocessing_status = 'failed',
+          preprocessing_finished_at = now(),
+          preprocessing_error_code = 'catalog_changed',
+          updated_at = now()
+      where id = resolved_database_id;
+      if tg_op = 'DELETE' then return old; else return new; end if;
+    end;
+    $$
+  `.execute(db);
+
+  for (const table of [
+    "catalog_tables",
+    "catalog_columns",
+    "catalog_relationships",
+    "catalog_relationship_columns",
+    "catalog_logical_relationships",
+    "database_bindings",
+  ]) {
+    await sql.raw(`
+      create trigger ${table}_metadata_write_guard
+      before insert or update or delete on ${table}
+      for each row execute function catalog_metadata_write_guard()
+    `).execute(db);
+  }
+
+  await sql`
+    create function catalog_database_configuration_guard()
+    returns trigger
+    language plpgsql
+    as $$
+    begin
+      if new.workspace_id is distinct from old.workspace_id
+        or new.engine is distinct from old.engine
+        or new.database_name is distinct from old.database_name
+        or new.schema_name is distinct from old.schema_name then
+        if old.preprocessing_status = 'running' then
+          raise exception 'catalog preprocessing is running'
+            using errcode = '55000';
+        end if;
+        new.metadata_content_revision := old.metadata_content_revision + 1;
+        new.preprocessing_status := 'failed';
+        new.preprocessing_finished_at := now();
+        new.preprocessing_error_code := 'catalog_changed';
+      end if;
+      return new;
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger workspace_databases_configuration_guard
+    before update of workspace_id, engine, database_name, schema_name on workspace_databases
+    for each row execute function catalog_database_configuration_guard()
+  `.execute(db);
+
+  await sql`
+    create function catalog_operation_start_guard()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      current_status text;
+      starting boolean;
+    begin
+      if tg_table_name = 'catalog_sync_runs' then
+        starting := new.state in ('queued', 'running', 'awaiting_confirmation', 'applying');
+      else
+        starting := new.status in ('queued', 'running');
+      end if;
+      if not starting then
+        return new;
+      end if;
+
+      select preprocessing_status into current_status
+      from workspace_databases
+      where id = new.database_id
+      for update;
+      if current_status = 'running' then
+        raise exception 'catalog preprocessing is running'
+          using errcode = '55000';
+      end if;
+      return new;
+    end;
+    $$
+  `.execute(db);
+
+  for (const table of [
+    "catalog_sync_runs",
+    "description_generation_runs",
+    "sensitive_data_suggestion_runs",
+  ]) {
+    await sql.raw(`
+      create trigger ${table}_operation_start_guard
+      before insert or update on ${table}
+      for each row execute function catalog_operation_start_guard()
+    `).execute(db);
+  }
+}
+
+export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+  for (const table of [
+    "catalog_sync_runs",
+    "description_generation_runs",
+    "sensitive_data_suggestion_runs",
+  ]) {
+    await sql.raw(`drop trigger if exists ${table}_operation_start_guard on ${table}`).execute(db);
+  }
+  await sql`drop function if exists catalog_operation_start_guard()`.execute(db);
+  await sql`drop trigger if exists workspace_databases_configuration_guard on workspace_databases`.execute(db);
+  await sql`drop function if exists catalog_database_configuration_guard()`.execute(db);
+  for (const table of [
+    "catalog_tables",
+    "catalog_columns",
+    "catalog_relationships",
+    "catalog_relationship_columns",
+    "catalog_logical_relationships",
+    "database_bindings",
+  ]) {
+    await sql.raw(`drop trigger if exists ${table}_metadata_write_guard on ${table}`).execute(db);
+  }
+  await sql`drop function if exists catalog_metadata_write_guard()`.execute(db);
+  await db.schema.alterTable("workspace_databases")
+    .dropConstraint("workspace_databases_preprocessing_status_check").execute();
+  for (const column of [
+    "preprocessing_error_code",
+    "preprocessing_finished_at",
+    "preprocessing_started_at",
+    "preprocessed_metadata_revision",
+    "preprocessing_input_fingerprint",
+    "preprocessing_status",
+    "metadata_content_revision",
+  ]) {
+    await sql.raw(`alter table workspace_databases drop column ${column}`).execute(db);
+  }
+}

backend/src/catalog/repository.ts

@@ -27,6 +27,8 @@ import {
   type CatalogLogicalRelationshipCandidate,
   type CatalogLogicalRelationshipContext,
   type CatalogPhysicalRelationship,
+  type CatalogPreprocessingStartResult,
+  type CatalogPreprocessingClearResult,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -45,15 +47,20 @@ import {
   type DescriptionGenerationScope,
   type ObservedCatalogTable,
   type ObservedSchemaSnapshot,
-  type SensitiveDataSuggestionEvent,
-  type SensitiveDataSuggestionRun,
-  type SensitiveDataSuggestionRunUpdate,
-  type SensitiveDataSuggestionScope,
+  type SensitivityAnalysisEvent,
+  type SensitivityAnalysisRun,
+  type SensitivityAnalysisRunUpdate,
+  type SensitivityAnalysisScope,
   type TableSyncRepositoryResult,
   type WorkspaceDatabase,
 } from "./types.js";
 
 type Timestamp = ColumnType<Date, Date | string | undefined, Date | string>;
+type NullableTimestamp = ColumnType<
+  Date | null,
+  Date | string | null | undefined,
+  Date | string | null
+>;
 
 interface WorkspaceDatabaseTable {
   id: string;
@@ -66,6 +73,13 @@ interface WorkspaceDatabaseTable {
   updatedAt: Timestamp;
   schemaSyncedVersion: number | null;
   schemaSyncedAt: Timestamp | null;
+  metadataContentRevision: Generated<number>;
+  preprocessingStatus: Generated<WorkspaceDatabase["preprocessingStatus"]>;
+  preprocessingInputFingerprint: Generated<string | null>;
+  preprocessedMetadataRevision: Generated<number | null>;
+  preprocessingStartedAt: NullableTimestamp;
+  preprocessingFinishedAt: NullableTimestamp;
+  preprocessingErrorCode: Generated<string | null>;
 }
 
 interface DatabaseBindingTable {
@@ -118,6 +132,7 @@ interface CatalogColumnTable {
   description: string | null;
   generatedDescription: string | null;
   sensitive: Generated<boolean>;
+  sensitivityReason: Generated<string | null>;
   lastSyncedDatabaseVersion: number | null;
   lastSyncedAt: Timestamp | null;
   version: Generated<number>;
@@ -189,15 +204,18 @@ interface DescriptionGenerationEventTable {
   createdAt: Timestamp;
 }
 
-interface SensitiveDataSuggestionRunTable {
+interface SensitivityAnalysisRunTable {
   id: string;
   databaseId: string;
-  scope: SensitiveDataSuggestionScope;
-  modelId: string;
-  status: SensitiveDataSuggestionRun["status"];
+  scope: SensitivityAnalysisScope;
+  engine: SensitivityAnalysisRun["engine"];
+  modelId: string | null;
+  policyVersion: string | null;
+  status: SensitivityAnalysisRun["status"];
   total: number;
   suggestedSensitive: number;
   suggestedNonSensitive: number;
+  unknown: number;
   inputTokens: number;
   cacheReadTokens: number;
   outputTokens: number;
@@ -208,10 +226,10 @@ interface SensitiveDataSuggestionRunTable {
   errorSummary: string | null;
 }
 
-interface SensitiveDataSuggestionEventTable {
+interface SensitivityAnalysisEventTable {
   runId: string;
   sequence: number;
-  level: SensitiveDataSuggestionEvent["level"];
+  level: SensitivityAnalysisEvent["level"];
   message: string;
   createdAt: Timestamp;
 }
@@ -264,8 +282,9 @@ export interface CatalogDatabase {
   catalogLogicalRelationships: CatalogLogicalRelationshipTable;
   descriptionGenerationRuns: DescriptionGenerationRunTable;
   descriptionGenerationEvents: DescriptionGenerationEventTable;
-  sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable;
-  sensitiveDataSuggestionEvents: SensitiveDataSuggestionEventTable;
+  // Legacy physical table names retained for migration and storage compatibility.
+  sensitiveDataSuggestionRuns: SensitivityAnalysisRunTable;
+  sensitiveDataSuggestionEvents: SensitivityAnalysisEventTable;
   catalogSyncRuns: CatalogSyncRunTable;
   catalogSyncEvents: CatalogSyncEventTable;
 }
@@ -320,6 +339,19 @@ function serialize(row: JoinedRow): WorkspaceDatabase {
     lastErrorMessage: present(row.lastErrorMessage),
     schemaSyncedVersion: present(row.schemaSyncedVersion),
     schemaSyncedAt: row.schemaSyncedAt == null ? undefined : new Date(row.schemaSyncedAt).toISOString(),
+    metadataContentRevision: Number(row.metadataContentRevision),
+    preprocessingStatus: row.preprocessingStatus,
+    preprocessingInputFingerprint: present(row.preprocessingInputFingerprint),
+    preprocessedMetadataRevision: row.preprocessedMetadataRevision == null
+      ? undefined
+      : Number(row.preprocessedMetadataRevision),
+    preprocessingStartedAt: row.preprocessingStartedAt == null
+      ? undefined
+      : new Date(row.preprocessingStartedAt).toISOString(),
+    preprocessingFinishedAt: row.preprocessingFinishedAt == null
+      ? undefined
+      : new Date(row.preprocessingFinishedAt).toISOString(),
+    preprocessingErrorCode: present(row.preprocessingErrorCode),
   };
 }
 
@@ -356,6 +388,7 @@ function serializeColumn(row: Selectable<CatalogColumnTable>, foreignKeyCount =
     description: row.description,
     generatedDescription: row.generatedDescription,
     sensitive: row.sensitive,
+    sensitivityReason: row.sensitivityReason,
     lastSyncedDatabaseVersion: row.lastSyncedDatabaseVersion,
     lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(),
     version: row.version,
@@ -402,9 +435,9 @@ function serializeDescriptionGenerationEvent(
   return { ...row, createdAt: new Date(row.createdAt).toISOString() };
 }
 
-function serializeSensitiveDataSuggestionRun(
-  row: Selectable<SensitiveDataSuggestionRunTable>,
-): SensitiveDataSuggestionRun {
+function serializeSensitivityAnalysisRun(
+  row: Selectable<SensitivityAnalysisRunTable>,
+): SensitivityAnalysisRun {
   const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString();
   return {
     ...row,
@@ -415,9 +448,9 @@ function serializeSensitiveDataSuggestionRun(
   };
 }
 
-function serializeSensitiveDataSuggestionEvent(
-  row: Selectable<SensitiveDataSuggestionEventTable>,
-): SensitiveDataSuggestionEvent {
+function serializeSensitivityAnalysisEvent(
+  row: Selectable<SensitivityAnalysisEventTable>,
+): SensitivityAnalysisEvent {
   return { ...row, createdAt: new Date(row.createdAt).toISOString() };
 }
 
@@ -471,6 +504,98 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return id ? await this.get(id.id) : undefined;
   }
 
+  async beginPreprocessing(
+    workspaceId: string,
+    inputFingerprint: string,
+  ): Promise<CatalogPreprocessingStartResult> {
+    return await this.db.transaction().execute(async (trx) => {
+      const database = await trx.selectFrom("workspaceDatabases")
+        .selectAll()
+        .where("workspaceId", "=", workspaceId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!database) return { kind: "not_found" };
+      if (database.preprocessingStatus === "running") return { kind: "already_running" };
+      if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
+
+      const activeSync = await trx.selectFrom("catalogSyncRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("state", "in", ["queued", "running", "awaiting_confirmation", "applying"])
+        .executeTakeFirst();
+      const activeDescriptions = await trx.selectFrom("descriptionGenerationRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("status", "in", ["queued", "running"])
+        .executeTakeFirst();
+      const activeSensitivity = await trx.selectFrom("sensitiveDataSuggestionRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("status", "=", "running")
+        .executeTakeFirst();
+      if (activeSync || activeDescriptions || activeSensitivity) return { kind: "catalog_busy" };
+
+      await trx.updateTable("workspaceDatabases")
+        .set({
+          preprocessingStatus: "running",
+          preprocessingInputFingerprint: inputFingerprint,
+          preprocessedMetadataRevision: null,
+          preprocessingStartedAt: sql`now()`,
+          preprocessingFinishedAt: null,
+          preprocessingErrorCode: null,
+          updatedAt: sql`now()`,
+        })
+        .where("id", "=", database.id)
+        .execute();
+      return { kind: "started", database: (await selectOne(trx, database.id))! };
+    });
+  }
+
+  async finishPreprocessing(
+    workspaceId: string,
+    metadataContentRevision: number,
+    inputFingerprint: string,
+    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
+  ): Promise<WorkspaceDatabase | undefined> {
+    const result = await this.db.updateTable("workspaceDatabases")
+      .set({
+        preprocessingStatus: outcome.status,
+        preprocessedMetadataRevision: outcome.status === "succeeded" ? metadataContentRevision : null,
+        preprocessingFinishedAt: sql`now()`,
+        preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : null,
+        updatedAt: sql`now()`,
+      })
+      .where("workspaceId", "=", workspaceId)
+      .where("preprocessingStatus", "=", "running")
+      .where("metadataContentRevision", "=", metadataContentRevision)
+      .where("preprocessingInputFingerprint", "=", inputFingerprint)
+      .returning("id")
+      .executeTakeFirst();
+    return result ? await this.get(result.id) : undefined;
+  }
+
+  async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
+    return await this.db.transaction().execute(async (trx) => {
+      const database = await trx.selectFrom("workspaceDatabases")
+        .select(["id", "preprocessingStatus"])
+        .where("workspaceId", "=", workspaceId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!database) return { kind: "not_found" };
+      if (database.preprocessingStatus === "running") return { kind: "already_running" };
+      await trx.updateTable("workspaceDatabases").set({
+        preprocessingStatus: "failed",
+        preprocessingInputFingerprint: null,
+        preprocessedMetadataRevision: null,
+        preprocessingStartedAt: null,
+        preprocessingFinishedAt: sql`now()`,
+        preprocessingErrorCode: "derived_data_cleared",
+        updatedAt: sql`now()`,
+      }).where("id", "=", database.id).execute();
+      return { kind: "cleared", database: (await selectOne(trx, database.id))! };
+    });
+  }
+
   async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
     const result = await sql<CatalogMetricsRow>`
       WITH requested_database AS (
@@ -512,10 +637,18 @@ export class KyselyCatalogRepository implements CatalogRepository {
       relationship_metrics AS (
         SELECT
           count(*)::int AS relationships,
-          max(catalog_relationships.updated_at) AS updated_at
-        FROM catalog_relationships
-        INNER JOIN selected_databases
-          ON selected_databases.id = catalog_relationships.database_id
+          max(relationship.updated_at) AS updated_at
+        FROM (
+          SELECT catalog_relationships.updated_at
+          FROM catalog_relationships
+          INNER JOIN selected_databases
+            ON selected_databases.id = catalog_relationships.database_id
+          UNION ALL
+          SELECT catalog_logical_relationships.updated_at
+          FROM catalog_logical_relationships
+          INNER JOIN selected_databases
+            ON selected_databases.id = catalog_logical_relationships.database_id
+        ) AS relationship
       )
       SELECT
         (SELECT count(*)::int FROM selected_databases) AS "databaseCount",
@@ -724,6 +857,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
     description: string | null,
     generatedDescription: string | null,
     sensitive?: boolean,
+    sensitivityReason?: string | null,
   ): Promise<CatalogColumn | undefined> {
     const belongs = await this.db.selectFrom("catalogTables").select("id")
       .where("id", "=", tableId).where("databaseId", "=", databaseId).executeTakeFirst();
@@ -732,6 +866,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
       description,
       generatedDescription,
       ...(sensitive === undefined ? {} : { sensitive }),
+      ...(sensitive === false
+        ? { sensitivityReason: null }
+        : sensitivityReason === undefined ? {} : { sensitivityReason }),
       version: sql`version + 1`,
       updatedAt: sql`now()`,
     }).where("id", "=", columnId).where("tableId", "=", tableId)
@@ -745,7 +882,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
     targetIds: readonly string[],
   ): Promise<CatalogDescriptionConsolidationCounts | undefined> {
     const selectedTargetIds = [...new Set(targetIds)];
-    if (selectedTargetIds.length === 0) return undefined;
+    if (target !== "database" && target !== "database_columns" && selectedTargetIds.length === 0) {
+      return undefined;
+    }
     return await this.db.transaction().execute(async (trx) => {
       const database = await trx.selectFrom("workspaceDatabases").select("id")
         .where("id", "=", databaseId).forUpdate().executeTakeFirst();
@@ -775,29 +914,51 @@ export class KyselyCatalogRepository implements CatalogRepository {
         };
       }
 
-      const tableRows = await trx.selectFrom("catalogTables").select("id")
-        .where("databaseId", "=", databaseId).execute();
-      const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
-        .select(["id", "generatedDescription"])
-        .where("tableId", "in", tableRows.map((table) => table.id))
-        .where("id", "in", selectedTargetIds)
+      const tableRows = await trx.selectFrom("catalogTables").select(["id", "generatedDescription"])
+        .where("databaseId", "=", databaseId)
         .orderBy("id")
         .forUpdate()
         .execute();
-      if (rows.length !== selectedTargetIds.length) return undefined;
+      const copiedTableIds = target === "database"
+        ? tableRows
+          .filter((row) => Boolean(row.generatedDescription?.trim()))
+          .map((row) => row.id)
+        : [];
+      if (copiedTableIds.length > 0) {
+        await trx.updateTable("catalogTables").set({
+          description: sql`generated_description`,
+          version: sql`version + 1`,
+          updatedAt: sql`now()`,
+        }).where("id", "in", copiedTableIds).execute();
+      }
+      const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
+        .select(["id", "generatedDescription"])
+        .where("tableId", "in", tableRows.map((table) => table.id))
+        .$if(target === "columns", (query) => query.where("id", "in", selectedTargetIds))
+        .orderBy("id")
+        .forUpdate()
+        .execute();
+      if (target === "columns" && rows.length !== selectedTargetIds.length) return undefined;
       const copiedIds = rows
         .filter((row) => Boolean(row.generatedDescription?.trim()))
         .map((row) => row.id);
       if (copiedIds.length > 0) {
-        await trx.updateTable("catalogColumns").set({
+        let update = trx.updateTable("catalogColumns").set({
           description: sql`generated_description`,
           version: sql`version + 1`,
           updatedAt: sql`now()`,
-        }).where("id", "in", copiedIds).execute();
+        });
+        update = target === "database" || target === "database_columns"
+          ? update
+            .where("tableId", "in", tableRows.map((table) => table.id))
+            .where(sql<boolean>`nullif(btrim(generated_description), '') is not null`)
+          : update.where("id", "in", copiedIds);
+        await update.execute();
       }
       return {
-        copied: copiedIds.length,
-        skipped: selectedTargetIds.length - copiedIds.length,
+        copied: copiedTableIds.length + copiedIds.length,
+        skipped: (target === "database" ? tableRows.length : 0) - copiedTableIds.length
+          + rows.length - copiedIds.length,
       };
     });
   }
@@ -938,52 +1099,55 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return rows.map(serializeDescriptionGenerationEvent);
   }
 
-  async createSensitiveDataSuggestionRun(
+  async createSensitivityAnalysisRun(
     databaseId: string,
-    scope: SensitiveDataSuggestionScope,
-    modelId: string,
-  ): Promise<SensitiveDataSuggestionRun> {
+    scope: SensitivityAnalysisScope,
+    origin: { engine: "llm"; modelId: string } | { engine: "local"; policyVersion: string },
+  ): Promise<SensitivityAnalysisRun> {
     const row = await this.db.insertInto("sensitiveDataSuggestionRuns").values({
       id: randomUUID(),
       databaseId,
       scope,
-      modelId,
+      engine: origin.engine,
+      modelId: origin.engine === "llm" ? origin.modelId : null,
+      policyVersion: origin.engine === "local" ? origin.policyVersion : null,
       status: "running",
       total: 0,
       suggestedSensitive: 0,
       suggestedNonSensitive: 0,
+      unknown: 0,
       inputTokens: 0,
       cacheReadTokens: 0,
       outputTokens: 0,
       finishedAt: null,
       errorSummary: null,
     }).returningAll().executeTakeFirstOrThrow();
-    return serializeSensitiveDataSuggestionRun(row);
+    return serializeSensitivityAnalysisRun(row);
   }
 
-  async getSensitiveDataSuggestionRun(
+  async getSensitivityAnalysisRun(
     runId: string,
-  ): Promise<SensitiveDataSuggestionRun | undefined> {
+  ): Promise<SensitivityAnalysisRun | undefined> {
     const row = await this.db.selectFrom("sensitiveDataSuggestionRuns")
       .selectAll()
       .where("id", "=", runId)
       .executeTakeFirst();
-    return row ? serializeSensitiveDataSuggestionRun(row) : undefined;
+    return row ? serializeSensitivityAnalysisRun(row) : undefined;
   }
 
-  async listSensitiveDataSuggestionRuns(limit = 50): Promise<SensitiveDataSuggestionRun[]> {
+  async listSensitivityAnalysisRuns(limit = 50): Promise<SensitivityAnalysisRun[]> {
     const rows = await this.db.selectFrom("sensitiveDataSuggestionRuns")
       .selectAll()
       .orderBy("createdAt", "desc")
       .orderBy("id", "desc")
       .limit(limit)
       .execute();
-    return rows.map(serializeSensitiveDataSuggestionRun);
+    return rows.map(serializeSensitivityAnalysisRun);
   }
 
-  async interruptActiveSensitiveDataSuggestionRuns(
+  async interruptActiveSensitivityAnalysisRuns(
     errorSummary: string,
-  ): Promise<SensitiveDataSuggestionRun[]> {
+  ): Promise<SensitivityAnalysisRun[]> {
     const rows = await this.db.updateTable("sensitiveDataSuggestionRuns")
       .set({
         status: "interrupted",
@@ -994,34 +1158,34 @@ export class KyselyCatalogRepository implements CatalogRepository {
       .where("status", "=", "running")
       .returningAll()
       .execute();
-    return rows.map(serializeSensitiveDataSuggestionRun);
+    return rows.map(serializeSensitivityAnalysisRun);
   }
 
-  async updateSensitiveDataSuggestionRun(
+  async updateSensitivityAnalysisRun(
     runId: string,
-    update: SensitiveDataSuggestionRunUpdate,
-  ): Promise<SensitiveDataSuggestionRun | undefined> {
+    update: SensitivityAnalysisRunUpdate,
+  ): Promise<SensitivityAnalysisRun | undefined> {
     const values: any = { ...update, updatedAt: sql`now()` };
     const row = await this.db.updateTable("sensitiveDataSuggestionRuns")
       .set(values)
       .where("id", "=", runId)
       .returningAll()
       .executeTakeFirst();
-    return row ? serializeSensitiveDataSuggestionRun(row) : undefined;
+    return row ? serializeSensitivityAnalysisRun(row) : undefined;
   }
 
-  async appendSensitiveDataSuggestionEvent(
+  async appendSensitivityAnalysisEvent(
     runId: string,
-    level: SensitiveDataSuggestionEvent["level"],
+    level: SensitivityAnalysisEvent["level"],
     message: string,
-  ): Promise<SensitiveDataSuggestionEvent> {
+  ): Promise<SensitivityAnalysisEvent> {
     return await this.db.transaction().execute(async (trx) => {
       const run = await trx.selectFrom("sensitiveDataSuggestionRuns")
         .select("id")
         .where("id", "=", runId)
         .forUpdate()
         .executeTakeFirst();
-      if (!run) throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist");
+      if (!run) throw new CatalogConflictError("Sensitivity Analysis Run does not exist");
       const current = await trx.selectFrom("sensitiveDataSuggestionEvents")
         .select(sql<number>`coalesce(max(sequence), 0)::int`.as("sequence"))
         .where("runId", "=", runId)
@@ -1032,21 +1196,21 @@ export class KyselyCatalogRepository implements CatalogRepository {
         level,
         message,
       }).returningAll().executeTakeFirstOrThrow();
-      return serializeSensitiveDataSuggestionEvent(row);
+      return serializeSensitivityAnalysisEvent(row);
     });
   }
 
-  async listSensitiveDataSuggestionEvents(
+  async listSensitivityAnalysisEvents(
     runId: string,
     afterSequence = 0,
-  ): Promise<SensitiveDataSuggestionEvent[]> {
+  ): Promise<SensitivityAnalysisEvent[]> {
     const rows = await this.db.selectFrom("sensitiveDataSuggestionEvents")
       .selectAll()
       .where("runId", "=", runId)
       .where("sequence", ">", afterSequence)
       .orderBy("sequence")
       .execute();
-    return rows.map(serializeSensitiveDataSuggestionEvent);
+    return rows.map(serializeSensitivityAnalysisEvent);
   }
 
   async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
@@ -1254,16 +1418,25 @@ export class KyselyCatalogRepository implements CatalogRepository {
       if (databases.length !== selectedDatabaseIds.length) return undefined;
 
       if (target === "relationships") {
-        const count = await trx.selectFrom("catalogRelationships")
+        const physicalCount = await trx.selectFrom("catalogRelationships")
           .select(sql<number>`count(*)::int`.as("count"))
           .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
+        const logicalCount = await trx.selectFrom("catalogLogicalRelationships")
+          .select(sql<number>`count(*)::int`.as("count"))
+          .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
+        await trx.deleteFrom("catalogLogicalRelationships")
+          .where("databaseId", "in", selectedDatabaseIds).execute();
         await trx.deleteFrom("catalogRelationships")
           .where("databaseId", "in", selectedDatabaseIds).execute();
         await trx.updateTable("workspaceDatabases").set({
           schemaSyncedVersion: null,
           schemaSyncedAt: null,
         }).where("id", "in", selectedDatabaseIds).execute();
-        return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
+        return {
+          tables: 0,
+          columns: 0,
+          relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0),
+        };
       }
 
       const tableCount = await trx.selectFrom("catalogTables")
@@ -1273,7 +1446,10 @@ export class KyselyCatalogRepository implements CatalogRepository {
         .innerJoin("catalogTables", "catalogTables.id", "catalogColumns.tableId")
         .select(sql<number>`count(*)::int`.as("count"))
         .where("catalogTables.databaseId", "in", selectedDatabaseIds).executeTakeFirst();
-      const relationshipCount = await trx.selectFrom("catalogRelationships")
+      const physicalRelationshipCount = await trx.selectFrom("catalogRelationships")
+        .select(sql<number>`count(*)::int`.as("count"))
+        .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
+      const logicalRelationshipCount = await trx.selectFrom("catalogLogicalRelationships")
         .select(sql<number>`count(*)::int`.as("count"))
         .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
       await trx.deleteFrom("catalogTables")
@@ -1285,7 +1461,8 @@ export class KyselyCatalogRepository implements CatalogRepository {
       return {
         tables: Number(tableCount?.count ?? 0),
         columns: Number(columnCount?.count ?? 0),
-        relationships: Number(relationshipCount?.count ?? 0),
+        relationships: Number(physicalRelationshipCount?.count ?? 0)
+          + Number(logicalRelationshipCount?.count ?? 0),
       };
     });
   }
@@ -1319,13 +1496,34 @@ export class KyselyCatalogRepository implements CatalogRepository {
         return { tables: 0, columns: Number(count?.count ?? 0), relationships: 0 };
       }
 
-      const count = await trx.selectFrom("catalogRelationships")
+      const physicalCount = await trx.selectFrom("catalogRelationships")
         .select(sql<number>`count(*)::int`.as("count"))
         .where("databaseId", "=", databaseId)
         .where((eb) => eb.or([
           eb("sourceTableId", "in", selectedTableIds),
           eb("targetTableId", "in", selectedTableIds),
         ])).executeTakeFirst();
+      const selectedColumnIds = (await trx.selectFrom("catalogColumns")
+        .select("id")
+        .where("tableId", "in", selectedTableIds)
+        .execute()).map((column) => column.id);
+      const logicalCount = selectedColumnIds.length === 0
+        ? undefined
+        : await trx.selectFrom("catalogLogicalRelationships")
+            .select(sql<number>`count(*)::int`.as("count"))
+            .where("databaseId", "=", databaseId)
+            .where((eb) => eb.or([
+              eb("sourceColumnId", "in", selectedColumnIds),
+              eb("targetColumnId", "in", selectedColumnIds),
+            ])).executeTakeFirst();
+      if (selectedColumnIds.length > 0) {
+        await trx.deleteFrom("catalogLogicalRelationships")
+          .where("databaseId", "=", databaseId)
+          .where((eb) => eb.or([
+            eb("sourceColumnId", "in", selectedColumnIds),
+            eb("targetColumnId", "in", selectedColumnIds),
+          ])).execute();
+      }
       await trx.deleteFrom("catalogRelationships")
         .where("databaseId", "=", databaseId)
         .where((eb) => eb.or([
@@ -1336,7 +1534,11 @@ export class KyselyCatalogRepository implements CatalogRepository {
         schemaSyncedVersion: null,
         schemaSyncedAt: null,
       }).where("id", "=", databaseId).execute();
-      return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
+      return {
+        tables: 0,
+        columns: 0,
+        relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0),
+      };
     });
   }
 
@@ -1402,6 +1604,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
     scope: CatalogSyncScope,
     tableIds: readonly string[],
     snapshot: ObservedSchemaSnapshot,
+    syncRunId?: string,
   ): Promise<CatalogSyncCounts | undefined> {
     return await this.db.transaction().execute(async (trx) => {
       const database = await trx.selectFrom("workspaceDatabases").select("version")
@@ -1577,6 +1780,10 @@ export class KyselyCatalogRepository implements CatalogRepository {
           schemaSyncedAt: now,
         }).where("id", "=", databaseId).execute();
       }
+      if (syncRunId) {
+        await trx.updateTable("catalogSyncRuns").set({ phase: "memory_cleanup" })
+          .where("id", "=", syncRunId).where("databaseId", "=", databaseId).execute();
+      }
       return {
         tables: snapshot.tables.length,
         columns: scope === "tables" ? undefined : snapshot.columns.length,
@@ -1680,7 +1887,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
 
   async interruptActiveSyncRuns(): Promise<void> {
     await this.db.updateTable("catalogSyncRuns").set({
-      state: "interrupted", phase: "completed", errorCode: "worker_restarted",
+      state: "interrupted", phase: sql`case when phase='memory_cleanup' then phase else 'completed' end`, errorCode: "worker_restarted",
       errorMessage: "Synchronization was interrupted by a backend restart.",
       finishedAt: sql`now()`, updatedAt: sql`now()`,
       leaseOwner: null, leaseExpiresAt: null,
@@ -1770,6 +1977,9 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async list(): Promise<WorkspaceDatabase[]> { return this.fail(); }
   async get(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
   async getByWorkspace(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
+  async beginPreprocessing(): Promise<CatalogPreprocessingStartResult> { return this.fail(); }
+  async finishPreprocessing(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
+  async clearPreprocessing(): Promise<CatalogPreprocessingClearResult> { return this.fail(); }
   async getCatalogMetrics(): Promise<CatalogMetrics | undefined> { return this.fail(); }
   async create(): Promise<WorkspaceDatabase> { return this.fail(); }
   async update(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
@@ -1792,13 +2002,13 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async updateDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
   async appendDescriptionGenerationEvent(): Promise<DescriptionGenerationEvent> { return this.fail(); }
   async listDescriptionGenerationEvents(): Promise<DescriptionGenerationEvent[]> { return this.fail(); }
-  async createSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun> { return this.fail(); }
-  async getSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
-  async listSensitiveDataSuggestionRuns(): Promise<SensitiveDataSuggestionRun[]> { return this.fail(); }
-  async interruptActiveSensitiveDataSuggestionRuns(): Promise<SensitiveDataSuggestionRun[]> { return this.fail(); }
-  async updateSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
-  async appendSensitiveDataSuggestionEvent(): Promise<SensitiveDataSuggestionEvent> { return this.fail(); }
-  async listSensitiveDataSuggestionEvents(): Promise<SensitiveDataSuggestionEvent[]> { return this.fail(); }
+  async createSensitivityAnalysisRun(): Promise<SensitivityAnalysisRun> { return this.fail(); }
+  async getSensitivityAnalysisRun(): Promise<SensitivityAnalysisRun | undefined> { return this.fail(); }
+  async listSensitivityAnalysisRuns(): Promise<SensitivityAnalysisRun[]> { return this.fail(); }
+  async interruptActiveSensitivityAnalysisRuns(): Promise<SensitivityAnalysisRun[]> { return this.fail(); }
+  async updateSensitivityAnalysisRun(): Promise<SensitivityAnalysisRun | undefined> { return this.fail(); }
+  async appendSensitivityAnalysisEvent(): Promise<SensitivityAnalysisEvent> { return this.fail(); }
+  async listSensitivityAnalysisEvents(): Promise<SensitivityAnalysisEvent[]> { return this.fail(); }
   async listRelationships(): Promise<CatalogPhysicalRelationship[]> { return this.fail(); }
   async listLogicalRelationships(): Promise<CatalogLogicalRelationship[]> { return this.fail(); }
   async getLogicalRelationshipContext(): Promise<CatalogLogicalRelationshipContext | undefined> { return this.fail(); }

backend/src/catalog/runtime-binding.ts

@@ -0,0 +1,140 @@
+import { dirname } from "node:path";
+
+import { resolveRuntimeBindings, type RuntimeBindings } from "../workspaces/bindings.js";
+import { buildInstallationContract, type InstallationSuffix } from "../workspaces/contracts.js";
+import {
+  discoverWorkspaceSecretRequirements,
+} from "../workspaces/secret-requirements.js";
+import type {
+  WorkspaceSecretMaterialization,
+  WorkspaceSecretStore,
+} from "../workspaces/secret-store.js";
+import {
+  validateWorkspaceDescriptor,
+  type WorkspaceDescriptor,
+} from "../workspaces/schema.js";
+import { CATALOG_SECRET_IDS } from "./secrets.js";
+import type { WorkspaceDatabase } from "./types.js";
+
+export interface CatalogRuntimeBindingLease {
+  workspace: WorkspaceDescriptor;
+  bindings: RuntimeBindings;
+  release(): void;
+}
+
+const CATALOG_SECRET_BY_SUFFIX: Readonly<Partial<Record<InstallationSuffix, string>>> = {
+  PASSWORD_FILE: CATALOG_SECRET_IDS.password,
+  API_KEY_FILE: CATALOG_SECRET_IDS.apiKey,
+  TLS_CA_FILE: CATALOG_SECRET_IDS.tlsCa,
+  SSH_PRIVATE_KEY_FILE: CATALOG_SECRET_IDS.sshPrivateKey,
+  SSH_KNOWN_HOSTS_FILE: CATALOG_SECRET_IDS.sshKnownHosts,
+};
+
+function runtimeWorkspace(
+  workspace: WorkspaceDescriptor,
+  database: WorkspaceDatabase,
+): WorkspaceDescriptor {
+  if (workspace.workspace.id !== database.workspaceId) {
+    throw new Error("Catalog database binding does not belong to the workspace");
+  }
+  const { dwh: _legacyDwh, diagnostics: _legacyDiagnostics, ...descriptor } = workspace;
+  const binding = database.binding;
+  return validateWorkspaceDescriptor({
+    ...descriptor,
+    dwh: {
+      engine: "postgres",
+      database: database.databaseName,
+      schema: database.schema,
+      ...(binding.port === undefined ? {} : { port: binding.port }),
+      supported_transports: [binding.transport],
+    },
+    ...(binding.transport === "rest_api" ? {
+      diagnostics: {
+        dwh_rest: {
+          method: "GET",
+          path: binding.restPath ?? "/health",
+          auth: binding.restAuth ?? "bearer",
+          response: { database: "database", schema: "schema" },
+        },
+      },
+    } : {}),
+  });
+}
+
+function setIfDefined(
+  environment: NodeJS.ProcessEnv,
+  name: string | undefined,
+  value: string | number | undefined,
+): void {
+  if (name !== undefined && value !== undefined && value !== "") environment[name] = String(value);
+}
+
+/**
+ * Project one PostgreSQL Catalog row into the legacy-shaped configuration consumed by the
+ * Python runtime. The authored workspace remains database-free; this object exists only for
+ * the lifetime of a backend-owned runtime lease.
+ */
+export function resolveCatalogRuntimeBinding(options: {
+  workspace: WorkspaceDescriptor;
+  database: WorkspaceDatabase;
+  environment: NodeJS.ProcessEnv;
+  secretRoots: readonly string[];
+  secretStore: WorkspaceSecretStore;
+}): CatalogRuntimeBindingLease {
+  const workspace = runtimeWorkspace(options.workspace, options.database);
+  const evidenceRequirements = discoverWorkspaceSecretRequirements(
+    options.workspace,
+    options.environment,
+  ).filter(({ connector }) => connector === "evidence");
+  const catalogSecretIds = Object.values(CATALOG_SECRET_IDS);
+  let materialization: WorkspaceSecretMaterialization | undefined;
+  try {
+    materialization = options.secretStore.materialize(
+      options.database.workspaceId,
+      [...catalogSecretIds, ...evidenceRequirements.map(({ id }) => id)],
+    );
+    const environment: NodeJS.ProcessEnv = { ...options.environment };
+    const roots = new Set(options.secretRoots);
+    for (const path of materialization.files.values()) roots.add(dirname(path));
+
+    const variables = buildInstallationContract(workspace).variables;
+    const variable = (role: "DWH" | "EVIDENCE", suffix: InstallationSuffix) => (
+      variables.find((candidate) => candidate.role === role && candidate.suffix === suffix)?.name
+    );
+    const binding = options.database.binding;
+    setIfDefined(environment, variable("DWH", "TRANSPORT"), binding.transport);
+    setIfDefined(environment, variable("DWH", "HOST"), binding.host);
+    setIfDefined(environment, variable("DWH", "PORT"), binding.port);
+    setIfDefined(environment, variable("DWH", "BASE_URL"), binding.baseUrl);
+    setIfDefined(environment, variable("DWH", "USER"), binding.username);
+    setIfDefined(environment, variable("DWH", "SSH_HOST"), binding.sshHost);
+    setIfDefined(environment, variable("DWH", "SSH_PORT"), binding.sshPort);
+    setIfDefined(environment, variable("DWH", "SSH_USER"), binding.sshUsername);
+    setIfDefined(environment, variable("DWH", "SSH_TARGET_HOST"), binding.sshTargetHost);
+    setIfDefined(environment, variable("DWH", "SSH_TARGET_PORT"), binding.sshTargetPort);
+    for (const [suffix, secretId] of Object.entries(CATALOG_SECRET_BY_SUFFIX) as Array<[
+      InstallationSuffix,
+      string,
+    ]>) {
+      setIfDefined(environment, variable("DWH", suffix), materialization.files.get(secretId));
+    }
+    for (const requirement of evidenceRequirements) {
+      setIfDefined(environment, requirement.variable, materialization.files.get(requirement.id));
+    }
+
+    const bindings = resolveRuntimeBindings(workspace, environment, [...roots]);
+    let released = false;
+    return {
+      workspace,
+      bindings,
+      release: () => {
+        if (released) return;
+        released = true;
+        materialization?.release();
+      },
+    };
+  } catch (error) {
+    materialization?.release();
+    throw error;
+  }
+}

backend/src/catalog/sensitive-data-suggester.ts

@@ -1,254 +0,0 @@
-import { z } from "zod";
-import type { MetadataGenerationModels } from "./metadata-generation-models.js";
-import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult, ModelCompletionUsage } from "./model-completer.js";
-import type {
-  CatalogColumn,
-  CatalogRepository,
-  CatalogTable,
-  SensitiveDataSuggestionScope,
-} from "./types.js";
-
-export type { SensitiveDataSuggestionScope } from "./types.js";
-
-// The helper accepts at most 64 KiB per message. Keep the same safety margin used by
-// Description Generation so UTF-8 structural metadata never reaches that hard limit.
-const MAX_USER_MESSAGE_BYTES = 60 * 1024;
-// Preserve ThothAI's proven completion granularity: small batches keep generation time and
-// structured-output accuracy predictable even when the helper byte limit would allow much more.
-const MAX_COLUMNS_PER_BATCH = 10;
-const responseSchema = z.object({
-  suggestions: z.array(z.object({
-    columnId: z.uuid(),
-    sensitive: z.boolean(),
-  }).strict()),
-}).strict();
-
-interface StructuralColumn {
-  columnId: string;
-  tableId: string;
-  table: string;
-  column: string;
-  dataType: string;
-  nullable: boolean;
-  primaryKey: boolean;
-  foreignKey: boolean;
-  version: number;
-  currentSensitive: boolean;
-}
-
-export interface SensitiveDataSuggestion {
-  columnId: string;
-  tableId: string;
-  tableName: string;
-  columnName: string;
-  version: number;
-  currentSensitive: boolean;
-  sensitive: boolean;
-}
-
-export class SensitiveDataSuggestionTargetNotFoundError extends Error {
-  constructor(readonly target: "database" | "table" | "column") {
-    super(`${target} not found`);
-    this.name = "SensitiveDataSuggestionTargetNotFoundError";
-  }
-}
-
-export class SensitiveDataSuggestionDuplicateTargetIdsError extends Error {
-  constructor() {
-    super("sensitive-data suggestion target IDs must be unique");
-    this.name = "SensitiveDataSuggestionDuplicateTargetIdsError";
-  }
-}
-
-export class SensitiveDataSuggestionNoEligibleColumnsError extends Error {
-  constructor(readonly scope: SensitiveDataSuggestionScope) {
-    super("selected scope has no catalog columns");
-    this.name = "SensitiveDataSuggestionNoEligibleColumnsError";
-  }
-}
-
-export class SensitiveDataSuggestionPayloadTooLargeError extends Error {
-  constructor() {
-    super("sensitive-data suggestion structural metadata is too large");
-    this.name = "SensitiveDataSuggestionPayloadTooLargeError";
-  }
-}
-
-export class SensitiveDataSuggestionInvalidResponseError extends Error {
-  constructor() {
-    super("sensitive-data suggestion response is invalid");
-    this.name = "SensitiveDataSuggestionInvalidResponseError";
-  }
-}
-
-function userContent(
-  database: { databaseName: string; schema: string },
-  columns: readonly StructuralColumn[],
-): string {
-  return JSON.stringify({
-    database: database.databaseName,
-    schema: database.schema,
-    columns: columns.map((column) => ({
-      columnId: column.columnId,
-      table: column.table,
-      column: column.column,
-      dataType: column.dataType,
-      nullable: column.nullable,
-      primaryKey: column.primaryKey,
-      foreignKey: column.foreignKey,
-    })),
-  });
-}
-
-function batchesFor(
-  database: { databaseName: string; schema: string },
-  columns: readonly StructuralColumn[],
-): StructuralColumn[][] {
-  const batches: StructuralColumn[][] = [];
-  let current: StructuralColumn[] = [];
-  for (const column of columns) {
-    if (current.length === MAX_COLUMNS_PER_BATCH) {
-      batches.push(current);
-      current = [];
-    }
-    const candidate = [...current, column];
-    if (Buffer.byteLength(userContent(database, candidate), "utf8") <= MAX_USER_MESSAGE_BYTES) {
-      current = candidate;
-      continue;
-    }
-    if (current.length === 0) throw new SensitiveDataSuggestionPayloadTooLargeError();
-    batches.push(current);
-    current = [column];
-    if (Buffer.byteLength(userContent(database, current), "utf8") > MAX_USER_MESSAGE_BYTES) {
-      throw new SensitiveDataSuggestionPayloadTooLargeError();
-    }
-  }
-  if (current.length > 0) batches.push(current);
-  return batches;
-}
-
-function structuralColumn(table: CatalogTable, column: CatalogColumn): StructuralColumn {
-  return {
-    columnId: column.id,
-    tableId: table.id,
-    table: table.name,
-    column: column.name,
-    dataType: column.dataType,
-    nullable: column.isNullable,
-    primaryKey: column.isPrimaryKey,
-    foreignKey: column.isForeignKey,
-    version: column.version,
-    currentSensitive: column.sensitive,
-  };
-}
-
-const systemMessage: ModelCompletionMessage = {
-  role: "system",
-  content: [
-    "Classify whether each database column is likely to contain sensitive source values.",
-    "Use only the supplied structural metadata. Return strict JSON with this exact shape:",
-    '{"suggestions":[{"columnId":"uuid","sensitive":true}]}',
-    "Return every supplied column exactly once. Do not add explanations or markdown.",
-  ].join("\n"),
-};
-
-export class SensitiveDataSuggester {
-  constructor(
-    private readonly repository: CatalogRepository,
-    private readonly models: MetadataGenerationModels,
-    private readonly completer: ModelCompleter,
-  ) {}
-
-  private async selectColumns(
-    databaseId: string,
-    scope: SensitiveDataSuggestionScope,
-    targetIds: readonly string[],
-  ): Promise<StructuralColumn[]> {
-    if (new Set(targetIds).size !== targetIds.length) {
-      throw new SensitiveDataSuggestionDuplicateTargetIdsError();
-    }
-    const tables = await this.repository.listTables(databaseId);
-    const tableIds = new Set(targetIds);
-    const selectedTables = scope === "selected_tables"
-      ? tables.filter((table) => tableIds.has(table.id))
-      : tables;
-    if (scope === "selected_tables" && selectedTables.length !== targetIds.length) {
-      throw new SensitiveDataSuggestionTargetNotFoundError("table");
-    }
-
-    const columns = (await Promise.all(selectedTables.map(async (table) => (
-      (await this.repository.listColumns(databaseId, table.id)).map((column) => (
-        structuralColumn(table, column)
-      ))
-    )))).flat();
-    const columnIds = new Set(targetIds);
-    const selectedColumns = scope === "selected_columns"
-      ? columns.filter((column) => columnIds.has(column.columnId))
-      : columns;
-    if (scope === "selected_columns" && selectedColumns.length !== targetIds.length) {
-      throw new SensitiveDataSuggestionTargetNotFoundError("column");
-    }
-    if (selectedColumns.length === 0) {
-      throw new SensitiveDataSuggestionNoEligibleColumnsError(scope);
-    }
-    return selectedColumns;
-  }
-
-  async suggest(
-    databaseId: string,
-    modelId: string,
-    scope: SensitiveDataSuggestionScope,
-    targetIds: readonly string[],
-    signal: AbortSignal,
-    onPrepared?: (total: number) => void | Promise<void>,
-    onProgress?: (processed: number, suggestions: readonly SensitiveDataSuggestion[]) => void | Promise<void>,
-    onUsage?: (usage: ModelCompletionUsage) => void | Promise<void>,
-  ): Promise<readonly SensitiveDataSuggestion[]> {
-    const database = await this.repository.get(databaseId);
-    if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
-    const columns = await this.selectColumns(databaseId, scope, targetIds);
-    await onPrepared?.(columns.length);
-    const model = this.models.resolve(modelId);
-    const suggestions: SensitiveDataSuggestion[] = [];
-
-    for (const batch of batchesFor(database, columns)) {
-      let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
-      for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
-        const completion = await this.completer.complete({
-          model,
-          signal,
-          messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
-        });
-        const result: ModelCompletionResult = typeof completion === "string"
-          ? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
-          : completion;
-        await onUsage?.(result.usage);
-        const content = result.content;
-        try {
-          const parsed = responseSchema.parse(JSON.parse(content));
-          const expected = new Set(batch.map((column) => column.columnId));
-          const candidate = new Map(parsed.suggestions.map((suggestion) => [suggestion.columnId, suggestion]));
-          if (candidate.size !== parsed.suggestions.length
-            || candidate.size !== expected.size
-            || [...candidate.keys()].some((columnId) => !expected.has(columnId))) {
-            throw new SensitiveDataSuggestionInvalidResponseError();
-          }
-          received = candidate;
-        } catch {
-          if (attempt === 1) throw new SensitiveDataSuggestionInvalidResponseError();
-        }
-      }
-      suggestions.push(...batch.map((column) => ({
-        columnId: column.columnId,
-        tableId: column.tableId,
-        tableName: column.table,
-        columnName: column.column,
-        version: column.version,
-        currentSensitive: column.currentSensitive,
-        sensitive: received!.get(column.columnId)!.sensitive,
-      })));
-      await onProgress?.(suggestions.length, suggestions.slice(-batch.length));
-    }
-    return suggestions;
-  }
-}

backend/src/catalog/sensitive-data-suggestion-runner.ts

@@ -1,136 +0,0 @@
-import type {
-  SensitiveDataSuggestion,
-} from "./sensitive-data-suggester.js";
-import {
-  SensitiveDataSuggester,
-  SensitiveDataSuggestionTargetNotFoundError,
-} from "./sensitive-data-suggester.js";
-import type {
-  CatalogRepository,
-  SensitiveDataSuggestionRun,
-  SensitiveDataSuggestionScope,
-} from "./types.js";
-import type { ModelCompletionUsage } from "./model-completer.js";
-
-const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
-const failedMessage = "Sensitive-field suggestion generation failed.";
-
-export interface SensitiveDataSuggestionRunResult {
-  suggestions: readonly SensitiveDataSuggestion[];
-  run: SensitiveDataSuggestionRun;
-}
-
-export class SensitiveDataSuggestionRunner {
-  constructor(
-    private readonly repository: CatalogRepository,
-    private readonly suggester: SensitiveDataSuggester,
-  ) {}
-
-  async initialize(): Promise<void> {
-    if (!(await this.repository.available())) return;
-    const interrupted = await this.repository.interruptActiveSensitiveDataSuggestionRuns(
-      interruptedMessage,
-    );
-    for (const run of interrupted) {
-      await this.repository.appendSensitiveDataSuggestionEvent(
-        run.id,
-        "warning",
-        interruptedMessage,
-      );
-    }
-  }
-
-  async run(
-    databaseId: string,
-    modelId: string,
-    scope: SensitiveDataSuggestionScope,
-    targetIds: readonly string[],
-    signal: AbortSignal,
-  ): Promise<SensitiveDataSuggestionRunResult> {
-    if (!(await this.repository.get(databaseId))) {
-      throw new SensitiveDataSuggestionTargetNotFoundError("database");
-    }
-    const started = await this.repository.createSensitiveDataSuggestionRun(
-      databaseId,
-      scope,
-      modelId,
-    );
-
-    try {
-      await this.repository.appendSensitiveDataSuggestionEvent(
-        started.id,
-        "info",
-        "Sensitive-field suggestion generation started.",
-      );
-      const suggestions = await this.suggester.suggest(
-        databaseId,
-        modelId,
-        scope,
-        targetIds,
-        signal,
-        async (total) => {
-          const prepared = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
-            total,
-          });
-          if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
-        },
-        async (processed, batch) => {
-          const suggestedSensitive = batch.filter((suggestion) => suggestion.sensitive).length;
-          const suggestedNonSensitive = batch.length - suggestedSensitive;
-          const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
-          if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
-          const progress = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
-            suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
-            suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
-          });
-          if (!progress) throw new Error("Sensitive Data Suggestion Run disappeared");
-          await this.repository.appendSensitiveDataSuggestionEvent(
-            started.id,
-            "info",
-            `Classified ${processed} of ${progress.total} columns.`,
-          );
-        },
-        async (usage: ModelCompletionUsage) => {
-          const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
-          if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
-          await this.repository.updateSensitiveDataSuggestionRun(started.id, {
-            inputTokens: current.inputTokens + usage.input,
-            cacheReadTokens: current.cacheReadTokens + usage.cacheRead,
-            outputTokens: current.outputTokens + usage.output,
-          });
-        },
-      );
-      const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
-      const suggestedNonSensitive = suggestions.length - suggestedSensitive;
-      await this.repository.appendSensitiveDataSuggestionEvent(
-        started.id,
-        "info",
-        `Sensitive-field suggestion generation completed for ${suggestions.length} column${
-          suggestions.length === 1 ? "" : "s"
-        }.`,
-      );
-      const completed = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
-        status: "completed",
-        total: suggestions.length,
-        suggestedSensitive,
-        suggestedNonSensitive,
-        finishedAt: new Date().toISOString(),
-        errorSummary: null,
-      });
-      if (!completed) throw new Error("Sensitive Data Suggestion Run disappeared");
-      return { suggestions, run: completed };
-    } catch (error) {
-      await this.repository.updateSensitiveDataSuggestionRun(started.id, {
-        status: "failed",
-        finishedAt: new Date().toISOString(),
-        errorSummary: failedMessage,
-      }).catch(() => undefined);
-      await this.repository.appendSensitiveDataSuggestionEvent(
-        started.id,
-        "error",
-        failedMessage,
-      ).catch(() => undefined);
-      throw error;
-    }
-  }
-}

backend/src/catalog/sensitivity-analysis-runner.ts

@@ -0,0 +1,178 @@
+import type {
+  SensitivityReviewItem,
+} from "./sensitivity-analysis-service.js";
+import {
+  SENSITIVITY_POLICY_VERSION,
+  SensitivityAnalysisInterruptedError,
+  SensitivityAnalysisService,
+  SensitivityAnalysisTargetNotFoundError,
+} from "./sensitivity-analysis-service.js";
+import type {
+  CatalogRepository,
+  SensitivityAnalysisRun,
+  SensitivityAnalysisScope,
+} from "./types.js";
+
+const interruptedMessage = "Local sensitivity analysis was interrupted by backend restart.";
+const interruptedDuringRunMessage = "Local sensitivity analysis was interrupted before completion.";
+const failedMessage = "Local sensitivity analysis failed.";
+
+function ensureActive(signal: AbortSignal): void {
+  if (signal.aborted) throw new SensitivityAnalysisInterruptedError();
+}
+
+export interface SensitivityAnalysisRunResult {
+  suggestions: readonly SensitivityReviewItem[];
+  run: SensitivityAnalysisRun;
+}
+
+export class SensitivityAnalysisRunner {
+  constructor(
+    private readonly repository: CatalogRepository,
+    private readonly analysis: SensitivityAnalysisService,
+  ) {}
+
+  async initialize(): Promise<void> {
+    if (!(await this.repository.available())) return;
+    const interrupted = await this.repository.interruptActiveSensitivityAnalysisRuns(
+      interruptedMessage,
+    );
+    for (const run of interrupted) {
+      await this.repository.appendSensitivityAnalysisEvent(
+        run.id,
+        "warning",
+        interruptedMessage,
+      );
+    }
+  }
+
+  async run(
+    databaseId: string,
+    scope: SensitivityAnalysisScope,
+    targetIds: readonly string[],
+    signal: AbortSignal,
+  ): Promise<SensitivityAnalysisRunResult> {
+    ensureActive(signal);
+    const database = await this.repository.get(databaseId);
+    ensureActive(signal);
+    if (!database) {
+      throw new SensitivityAnalysisTargetNotFoundError("database");
+    }
+    ensureActive(signal);
+    const started = await this.repository.createSensitivityAnalysisRun(
+      databaseId,
+      scope,
+      { engine: "local", policyVersion: SENSITIVITY_POLICY_VERSION },
+    );
+    let preparedTotal = 0;
+    let processedSensitive = 0;
+    let processedNonSensitive = 0;
+
+    try {
+      ensureActive(signal);
+      await this.repository.appendSensitivityAnalysisEvent(
+        started.id,
+        "info",
+        "Local sensitivity analysis started.",
+      );
+      ensureActive(signal);
+      const suggestions = await this.analysis.analyze(
+        databaseId,
+        scope,
+        targetIds,
+        signal,
+        async (total) => {
+          ensureActive(signal);
+          preparedTotal = total;
+          const prepared = await this.repository.updateSensitivityAnalysisRun(started.id, {
+            total,
+          });
+          ensureActive(signal);
+          if (!prepared) throw new Error("Sensitivity Analysis Run disappeared");
+        },
+        async (processed, batch) => {
+          ensureActive(signal);
+          const suggestedSensitive = batch.filter(
+            (suggestion) => suggestion.assessment === "sensitive",
+          ).length;
+          const suggestedNonSensitive = batch.filter(
+            (suggestion) => suggestion.assessment === "non_sensitive",
+          ).length;
+          const current = await this.repository.getSensitivityAnalysisRun(started.id);
+          ensureActive(signal);
+          if (!current) throw new Error("Sensitivity Analysis Run disappeared");
+          const progress = await this.repository.updateSensitivityAnalysisRun(started.id, {
+            suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
+            suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
+          });
+          if (!progress) throw new Error("Sensitivity Analysis Run disappeared");
+          processedSensitive += suggestedSensitive;
+          processedNonSensitive += suggestedNonSensitive;
+          ensureActive(signal);
+          await this.repository.appendSensitivityAnalysisEvent(
+            started.id,
+            "info",
+            `Assessed ${processed} of ${progress.total} columns locally.`,
+          );
+          ensureActive(signal);
+        },
+        async (message) => {
+          ensureActive(signal);
+          await this.repository.appendSensitivityAnalysisEvent(
+            started.id,
+            "info",
+            message,
+          );
+          ensureActive(signal);
+        },
+      );
+      ensureActive(signal);
+      const suggestedSensitive = suggestions.filter(
+        (suggestion) => suggestion.assessment === "sensitive",
+      ).length;
+      const suggestedNonSensitive = suggestions.filter(
+        (suggestion) => suggestion.assessment === "non_sensitive",
+      ).length;
+      await this.repository.appendSensitivityAnalysisEvent(
+        started.id,
+        "info",
+        `Local sensitivity analysis completed for ${suggestions.length} column${
+          suggestions.length === 1 ? "" : "s"
+        }.`,
+      );
+      ensureActive(signal);
+      const completed = await this.repository.updateSensitivityAnalysisRun(started.id, {
+        status: "completed",
+        total: suggestions.length,
+        suggestedSensitive,
+        suggestedNonSensitive,
+        unknown: 0,
+        finishedAt: new Date().toISOString(),
+        errorSummary: null,
+      });
+      ensureActive(signal);
+      if (!completed) throw new Error("Sensitivity Analysis Run disappeared");
+      return { suggestions, run: completed };
+    } catch (error) {
+      const interrupted = signal.aborted || error instanceof SensitivityAnalysisInterruptedError;
+      const message = interrupted ? interruptedDuringRunMessage : failedMessage;
+      await this.repository.updateSensitivityAnalysisRun(started.id, {
+        status: interrupted ? "interrupted" : "failed",
+        ...(interrupted ? {
+          total: preparedTotal,
+          suggestedSensitive: processedSensitive,
+          suggestedNonSensitive: processedNonSensitive,
+          unknown: Math.max(0, preparedTotal - processedSensitive - processedNonSensitive),
+        } : {}),
+        finishedAt: new Date().toISOString(),
+        errorSummary: message,
+      }).catch(() => undefined);
+      await this.repository.appendSensitivityAnalysisEvent(
+        started.id,
+        interrupted ? "warning" : "error",
+        message,
+      ).catch(() => undefined);
+      throw error;
+    }
+  }
+}

backend/src/catalog/sensitivity-analysis-service.ts

@@ -0,0 +1,184 @@
+import type {
+  SensitivityClassifier,
+  SensitivityColumnAssessment,
+  SensitivityEvidence,
+  SensitivityNerBudget,
+} from "./sensitivity-classifier.js";
+import type {
+  CatalogColumn,
+  CatalogRepository,
+  CatalogTable,
+  SensitivityAnalysisScope,
+} from "./types.js";
+
+export type { SensitivityAnalysisScope } from "./types.js";
+export const SENSITIVITY_POLICY_VERSION = "sensitivity-v4";
+
+interface SelectedColumn {
+  table: CatalogTable;
+  column: CatalogColumn;
+}
+
+export interface SensitivityReviewItem {
+  columnId: string;
+  tableId: string;
+  tableName: string;
+  columnName: string;
+  version: number;
+  currentSensitive: boolean;
+  sensitive: boolean;
+  assessment: SensitivityColumnAssessment["assessment"];
+  evidence: readonly SensitivityEvidence[];
+  observedValues: number;
+  coverage: SensitivityColumnAssessment["coverage"];
+}
+
+export class SensitivityAnalysisTargetNotFoundError extends Error {
+  constructor(readonly target: "database" | "table" | "column") {
+    super(`${target} not found`);
+    this.name = "SensitivityAnalysisTargetNotFoundError";
+  }
+}
+
+export class SensitivityAnalysisDuplicateTargetIdsError extends Error {
+  constructor() {
+    super("sensitivity analysis target IDs must be unique");
+    this.name = "SensitivityAnalysisDuplicateTargetIdsError";
+  }
+}
+
+export class SensitivityAnalysisInterruptedError extends Error {
+  constructor() {
+    super("sensitivity analysis interrupted");
+    this.name = "SensitivityAnalysisInterruptedError";
+  }
+}
+
+function ensureActive(signal: AbortSignal): void {
+  if (signal.aborted) throw new SensitivityAnalysisInterruptedError();
+}
+
+export class SensitivityAnalysisNoEligibleColumnsError extends Error {
+  constructor(readonly scope: SensitivityAnalysisScope) {
+    super("selected scope has no catalog columns");
+    this.name = "SensitivityAnalysisNoEligibleColumnsError";
+  }
+}
+
+/** Selection and table orchestration around the single SensitivityClassifier decision module. */
+export class SensitivityAnalysisService {
+  constructor(
+    private readonly repository: CatalogRepository,
+    private readonly classifier: SensitivityClassifier,
+    private readonly options: { nerBudgetMs?: number } = {},
+  ) {}
+
+  private async selectColumns(
+    databaseId: string,
+    scope: SensitivityAnalysisScope,
+    targetIds: readonly string[],
+    signal: AbortSignal,
+  ): Promise<readonly SelectedColumn[]> {
+    ensureActive(signal);
+    if (new Set(targetIds).size !== targetIds.length) {
+      throw new SensitivityAnalysisDuplicateTargetIdsError();
+    }
+    const tables = await this.repository.listTables(databaseId);
+    ensureActive(signal);
+    const tableIds = new Set(targetIds);
+    const selectedTables = scope === "selected_tables"
+      ? tables.filter((table) => tableIds.has(table.id))
+      : tables;
+    if (scope === "selected_tables" && selectedTables.length !== targetIds.length) {
+      throw new SensitivityAnalysisTargetNotFoundError("table");
+    }
+    const columns = (await Promise.all(selectedTables.map(async (table) => (
+      (await this.repository.listColumns(databaseId, table.id)).map((column) => ({ table, column }))
+    )))).flat();
+    ensureActive(signal);
+    const columnIds = new Set(targetIds);
+    const selectedColumns = scope === "selected_columns"
+      ? columns.filter(({ column }) => columnIds.has(column.id))
+      : columns;
+    if (scope === "selected_columns" && selectedColumns.length !== targetIds.length) {
+      throw new SensitivityAnalysisTargetNotFoundError("column");
+    }
+    if (selectedColumns.length === 0) {
+      throw new SensitivityAnalysisNoEligibleColumnsError(scope);
+    }
+    return selectedColumns;
+  }
+
+  async analyze(
+    databaseId: string,
+    scope: SensitivityAnalysisScope,
+    targetIds: readonly string[],
+    signal: AbortSignal,
+    onPrepared?: (total: number) => void | Promise<void>,
+    onProgress?: (processed: number, suggestions: readonly SensitivityReviewItem[]) => void | Promise<void>,
+    onActivity?: (message: string) => void | Promise<void>,
+  ): Promise<readonly SensitivityReviewItem[]> {
+    const configuredNerBudget = this.options.nerBudgetMs ?? 10_000;
+    const nerBudget: SensitivityNerBudget = {
+      remainingMs: Number.isFinite(configuredNerBudget) && configuredNerBudget >= 0
+        ? configuredNerBudget
+        : 10_000,
+    };
+    ensureActive(signal);
+    const database = await this.repository.get(databaseId);
+    ensureActive(signal);
+    if (!database) throw new SensitivityAnalysisTargetNotFoundError("database");
+    const selected = await this.selectColumns(databaseId, scope, targetIds, signal);
+    await onPrepared?.(selected.length);
+    ensureActive(signal);
+    const byTable = new Map<string, SelectedColumn[]>();
+    for (const item of selected) {
+      const items = byTable.get(item.table.id) ?? [];
+      items.push(item);
+      byTable.set(item.table.id, items);
+    }
+    const tableTargets = [...byTable.values()].map((items) => {
+      const first = items[0]!;
+      return {
+        database,
+        table: first.table,
+        columns: items.map(({ column }) => column),
+      };
+    });
+    const assessments = await this.classifier.assess(
+      tableTargets,
+      signal,
+      nerBudget,
+      onActivity,
+    );
+    ensureActive(signal);
+    const assessmentById = new Map(assessments.map((assessment) => [
+      assessment.columnId,
+      assessment,
+    ]));
+    const suggestions: SensitivityReviewItem[] = [];
+    for (const items of byTable.values()) {
+      ensureActive(signal);
+      const batch = items.map(({ table, column }) => {
+        const assessment = assessmentById.get(column.id)!;
+        return {
+          columnId: column.id,
+          tableId: table.id,
+          tableName: table.name,
+          columnName: column.name,
+          version: column.version,
+          currentSensitive: column.sensitive,
+          sensitive: assessment.proposedSensitive,
+          assessment: assessment.assessment,
+          evidence: assessment.evidence,
+          observedValues: assessment.observedValues,
+          coverage: assessment.coverage,
+        };
+      });
+      suggestions.push(...batch);
+      await onProgress?.(suggestions.length, batch);
+      ensureActive(signal);
+    }
+    return suggestions;
+  }
+}

backend/src/catalog/sensitivity-classifier.ts

@@ -0,0 +1,535 @@
+import type { CatalogColumn, CatalogTable, WorkspaceDatabase } from "./types.js";
+import { findPhoneNumbersInText } from "libphonenumber-js/max";
+import validator from "validator";
+
+export type SensitivityAssessment = "sensitive" | "non_sensitive";
+
+export interface SensitivityEvidence {
+  kind: "metadata" | "content" | "length" | "ner" | "coverage" | "type";
+  ruleId: string;
+  label?: string;
+  confidence?: number;
+}
+
+export interface SensitivityValueObservation {
+  columnId: string;
+  value: string | null;
+  characterLength: number | null;
+}
+
+export interface SensitivityScanCoverage {
+  kind: "complete" | "sampled";
+  observedValues: number;
+}
+
+export interface SensitivityTableScan {
+  batches: readonly (readonly SensitivityValueObservation[])[];
+  coverage: SensitivityScanCoverage;
+}
+
+export interface SensitivityScanRequest {
+  database: WorkspaceDatabase;
+  table: CatalogTable;
+  columns: readonly CatalogColumn[];
+  valuesPerColumn: number;
+  sampleOffset: number;
+  sampleSeed: number;
+  queryTimeoutMs: number;
+  fullScanThreshold?: number;
+}
+
+export interface SensitivityValueSource {
+  scanTable(
+    request: SensitivityScanRequest,
+    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
+    signal: AbortSignal,
+  ): Promise<SensitivityScanCoverage>;
+}
+
+export interface LocalNerCandidate {
+  columnId: string;
+  text: string;
+}
+
+export interface LocalNerEvidence {
+  columnId: string;
+  label: string;
+  confidence: number;
+}
+
+export interface SensitivityNerBudget {
+  remainingMs: number;
+}
+
+/** Optional local detector. It returns evidence only; it never decides a column assessment. */
+export interface LocalNerDetector {
+  warmup?(): Promise<void>;
+  isReady?(): boolean;
+  detect(
+    candidates: readonly LocalNerCandidate[],
+    signal: AbortSignal,
+    deadline: number,
+  ): Promise<readonly LocalNerEvidence[]>;
+  close?(): Promise<void>;
+}
+
+export interface SensitivityColumnAssessment {
+  columnId: string;
+  assessment: SensitivityAssessment;
+  proposedSensitive: boolean;
+  evidence: readonly SensitivityEvidence[];
+  observedValues: number;
+  coverage: "metadata" | "complete" | "sampled" | "no_values";
+}
+
+export interface SensitivityTableTarget {
+  database: WorkspaceDatabase;
+  table: CatalogTable;
+  columns: readonly CatalogColumn[];
+}
+
+const EMAIL = /(?<![\p{L}\p{N}._%+-])[\p{L}\p{N}._%+-]+@[\p{L}\p{N}.-]+\.[\p{L}]{2,63}(?![\p{L}\p{N}._%+-])/giu;
+const DIRECT_IDENTIFIER_NAMES = new Set([
+  "address", "birth_date", "codice_fiscale", "date_of_birth", "dob", "email", "e_mail",
+  "bic", "first_name", "fiscal_code", "full_name", "iban", "indirizzo", "last_name", "mobile",
+  "nome", "passport", "phone", "surname", "swift", "swift_code", "tax_id", "telefono",
+]);
+const CREDENTIAL_NAME = /(?:^|_)(?:api_key|credential|password|passwd|private_key|pwd|secret|token)(?:_|$)/u;
+const HEALTH_NAME = /(?:^|_)(?:anamnesi|clinical|diagnos(?:i|is)|health|medical|patient|patologia|therapy|terapia)(?:_|$)/u;
+const CLINICAL_TERM = /(?:^|[^\p{L}])(?:allergi[ae]|anamnesi|carcinoma|chemioterapia|diabete|diagnos[ei]|epatite|farmac[io]|gravidanza|hiv|metastasi|neoplasia|patologia|radioterapia|referto|terapia|tumore)(?:$|[^\p{L}])/iu;
+const UNSUPPORTED_BINARY_TYPE = /(?:^|\s)(?:binary|blob|bytea|image|varbinary)(?:\s|$|\()/iu;
+const DEEP_TEXT_TYPE = /(?:^|\s)(?:char|character|citext|clob|json|jsonb|nchar|nvarchar|string|text|varchar|xml)(?:\s|$|\()/iu;
+const MAX_NER_CANDIDATES_PER_REQUEST = 128;
+const MAX_CONCURRENT_TABLE_SCANS = 2;
+
+export const SENSITIVITY_SAMPLE_PHASES = [
+  { targetValuesPerColumn: 300, additionalValuesPerColumn: 300, sampleSeed: 37, deepTextOnly: false },
+  { targetValuesPerColumn: 1_000, additionalValuesPerColumn: 700, sampleSeed: 73, deepTextOnly: false },
+  { targetValuesPerColumn: 3_000, additionalValuesPerColumn: 2_000, sampleSeed: 109, deepTextOnly: true },
+] as const;
+
+function normalizedName(value: string): string {
+  return value.normalize("NFKD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .toLocaleLowerCase("en-US")
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "");
+}
+
+function boundedCount(value: number | undefined, fallback: number, maximum: number): number {
+  return value === undefined || !Number.isSafeInteger(value)
+    ? fallback
+    : Math.max(1, Math.min(value, maximum));
+}
+
+function metadataEvidence(column: CatalogColumn): SensitivityEvidence | undefined {
+  const ruleId = sensitiveNameRule(column.name);
+  return ruleId ? { kind: "metadata", ruleId } : undefined;
+}
+
+function nonSensitiveStructuralEvidence(column: CatalogColumn): SensitivityEvidence | undefined {
+  if (column.dataType.trim().toLowerCase() !== "bigint") return undefined;
+  if (column.isPrimaryKey || column.primaryKeyPosition !== null) {
+    return {
+      kind: "type",
+      ruleId: "type.bigint_primary_key_non_informative",
+      label: "non-informative bigint primary key",
+    };
+  }
+  if (normalizedName(column.name) === "pk") {
+    return {
+      kind: "metadata",
+      ruleId: "metadata.bigint_pk_identifier_non_informative",
+      label: "non-informative conventional bigint primary-key identifier",
+    };
+  }
+  return undefined;
+}
+
+function sensitiveNameRule(value: string): string | undefined {
+  const name = normalizedName(value);
+  if (DIRECT_IDENTIFIER_NAMES.has(name)) {
+    return "metadata.direct_identifier";
+  }
+  if (CREDENTIAL_NAME.test(name)) {
+    return "metadata.credential";
+  }
+  if (HEALTH_NAME.test(name)) {
+    return "metadata.health";
+  }
+  return undefined;
+}
+
+const ITALIAN_FISCAL_CODE = /(?<![A-Z0-9])[A-Z]{6}[0-9LMNPQRSTUV]{2}[ABCDEHLMPRST][0-9LMNPQRSTUV]{2}[A-Z][0-9LMNPQRSTUV]{3}[A-Z](?![A-Z0-9])/giu;
+const FISCAL_ODD: Record<string, number> = {
+  "0": 1, "1": 0, "2": 5, "3": 7, "4": 9, "5": 13, "6": 15, "7": 17, "8": 19, "9": 21,
+  A: 1, B: 0, C: 5, D: 7, E: 9, F: 13, G: 15, H: 17, I: 19, J: 21,
+  K: 2, L: 4, M: 18, N: 20, O: 11, P: 3, Q: 6, R: 8, S: 12, T: 14,
+  U: 16, V: 10, W: 22, X: 25, Y: 24, Z: 23,
+};
+
+function validItalianFiscalCode(candidate: string): boolean {
+  const value = candidate.toUpperCase();
+  if (value.length !== 16) return false;
+  let sum = 0;
+  for (let index = 0; index < 15; index += 1) {
+    const character = value[index]!;
+    if (index % 2 === 0) sum += FISCAL_ODD[character] ?? -1000;
+    else sum += /\d/u.test(character) ? Number(character) : character.charCodeAt(0) - 65;
+  }
+  return String.fromCharCode(65 + (sum % 26)) === value[15];
+}
+
+function validIban(candidate: string): boolean {
+  const value = candidate.replace(/\s+/gu, "").toUpperCase();
+  if (!/^[A-Z]{2}\d{2}[A-Z0-9]{11,30}$/u.test(value)) return false;
+  const rearranged = value.slice(4) + value.slice(0, 4);
+  let remainder = 0;
+  for (const character of rearranged) {
+    const digits = /\d/u.test(character) ? character : String(character.charCodeAt(0) - 55);
+    for (const digit of digits) remainder = (remainder * 10 + Number(digit)) % 97;
+  }
+  return remainder === 1;
+}
+
+function validPaymentCard(candidate: string): boolean {
+  const digits = candidate.replace(/[ -]/gu, "");
+  if (!/^\d{13,19}$/u.test(digits) || /^(\d)\1+$/u.test(digits)) return false;
+  let sum = 0;
+  let double = false;
+  for (let index = digits.length - 1; index >= 0; index -= 1) {
+    let digit = Number(digits[index]);
+    if (double) {
+      digit *= 2;
+      if (digit > 9) digit -= 9;
+    }
+    sum += digit;
+    double = !double;
+  }
+  return sum % 10 === 0;
+}
+
+function jsonHasSensitiveKey(value: string): boolean {
+  const trimmed = value.trim();
+  if (!(trimmed.startsWith("{") || trimmed.startsWith("["))) return false;
+  try {
+    const pending: Array<{ value: unknown; depth: number }> = [{ value: JSON.parse(trimmed), depth: 0 }];
+    let visited = 0;
+    while (pending.length > 0 && visited < 1_000) {
+      const item = pending.pop()!;
+      visited += 1;
+      if (item.depth > 8 || item.value === null || typeof item.value !== "object") continue;
+      if (Array.isArray(item.value)) {
+        for (const child of item.value) pending.push({ value: child, depth: item.depth + 1 });
+        continue;
+      }
+      for (const [key, child] of Object.entries(item.value)) {
+        if (sensitiveNameRule(key)) return true;
+        pending.push({ value: child, depth: item.depth + 1 });
+      }
+    }
+  } catch {
+    return false;
+  }
+  return false;
+}
+
+function contentEvidence(value: string): SensitivityEvidence | undefined {
+  if (/-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----/u.test(value)) {
+    return { kind: "content", ruleId: "credential.private_key" };
+  }
+  if (/(?:^|[^A-Z0-9])AKIA[A-Z0-9]{16}(?![A-Z0-9])/u.test(value)
+    || /(?:^|[^A-Za-z0-9_])gh[pousr]_[A-Za-z0-9_]{30,}(?![A-Za-z0-9_])/u.test(value)
+    || /(?:^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}(?![A-Za-z0-9_-])/u.test(value)) {
+    return { kind: "content", ruleId: "credential.access_key" };
+  }
+  if (/(?:^|[^\p{L}\p{N}_])(?:api[_ -]?key|access[_ -]?token|password|passwd|pwd|secret)\s*[:=]\s*[^\s,;]{4,}/iu.test(value)) {
+    return { kind: "content", ruleId: "credential.key_value" };
+  }
+  if (CLINICAL_TERM.test(value)) return { kind: "content", ruleId: "health.clinical_term" };
+  for (const match of value.matchAll(EMAIL)) {
+    if (validator.isEmail(match[0])) return { kind: "content", ruleId: "pii.email" };
+  }
+  for (const match of value.matchAll(ITALIAN_FISCAL_CODE)) {
+    if (validItalianFiscalCode(match[0])) {
+      return { kind: "content", ruleId: "pii.italian_fiscal_code" };
+    }
+  }
+  for (const match of value.matchAll(/\b(?:passaporto|passport)(?:\s+(?:numero|number|n\.?))?\s*[:#-]?\s*([A-Z0-9]{9})\b/giu)) {
+    if (validator.isPassportNumber(match[1]!, "IT")) {
+      return { kind: "content", ruleId: "pii.passport_number" };
+    }
+  }
+  for (const match of value.matchAll(/\bC[A-Z]\d{5}[A-Z]{2}\b/giu)) {
+    if (validator.isIdentityCard(match[0], "IT")) {
+      return { kind: "content", ruleId: "pii.identity_card" };
+    }
+  }
+  if (/\b(?:patente(?:\s+di\s+guida)?|driving\s+licen[cs]e)(?:\s+(?:numero|number|n\.?))?\s*[:#-]?\s*[A-Z0-9]{8,12}\b/iu.test(value)) {
+    return { kind: "content", ruleId: "pii.drivers_license_number" };
+  }
+  for (const match of value.matchAll(/(?<![A-Z0-9])[A-Z]{2}\d{2}(?:\s?[A-Z0-9]){11,30}(?![A-Z0-9])/giu)) {
+    if (validIban(match[0])) return { kind: "content", ruleId: "financial.iban" };
+  }
+  for (const match of value.matchAll(/(?<!\d)(?:\d[ -]?){13,19}(?!\d)/gu)) {
+    if (validPaymentCard(match[0])) {
+      return { kind: "content", ruleId: "financial.payment_card" };
+    }
+  }
+  for (const match of value.matchAll(/(?<![A-Z0-9])[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?(?![A-Z0-9])/giu)) {
+    const before = value.slice(Math.max(0, (match.index ?? 0) - 24), match.index ?? 0);
+    if (/\b(?:bic|swift)\s*[:=-]?\s*$/iu.test(before) && validator.isBIC(match[0])) {
+      return { kind: "content", ruleId: "financial.bic" };
+    }
+  }
+  for (const match of value.matchAll(/(?<!\d)(?:IT[ .-]?)?\d{11}(?!\d)/giu)) {
+    const candidate = match[0].replace(/[ .-]/gu, "");
+    if (validator.isVAT(candidate.replace(/^IT/iu, ""), "IT")) {
+      return { kind: "content", ruleId: "pii.italian_vat" };
+    }
+  }
+  for (const match of value.matchAll(/(?<![A-F0-9])(?:[A-F0-9]{2}[:-]){5}[A-F0-9]{2}(?![A-F0-9])/giu)) {
+    if (validator.isMACAddress(match[0])) {
+      return { kind: "content", ruleId: "network.mac_address" };
+    }
+  }
+  for (const match of value.matchAll(/(?<![A-F0-9:.])[A-F0-9:.]{3,45}(?![A-F0-9:.])/giu)) {
+    if (validator.isIP(match[0])) return { kind: "content", ruleId: "network.ip_address" };
+  }
+  for (const match of value.matchAll(/(?<![A-F0-9-])[0-9A-F]{8}-[0-9A-F]{4}-[1-8][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}(?![A-F0-9-])/giu)) {
+    if (validator.isUUID(match[0])) return { kind: "content", ruleId: "pii.uuid" };
+  }
+  for (const match of value.matchAll(/\b(?:https?|ftp):\/\/[^\s<>"']+/giu)) {
+    const candidate = match[0].replace(/[.,;:!?\])}]+$/u, "");
+    if (validator.isURL(candidate, { require_protocol: true })) {
+      return { kind: "content", ruleId: "network.url" };
+    }
+  }
+  if (findPhoneNumbersInText(value, "IT").some((match) => match.number.isValid())) {
+    return { kind: "content", ruleId: "pii.phone_number" };
+  }
+  if (jsonHasSensitiveKey(value)) {
+    return { kind: "content", ruleId: "pii.json_sensitive_key" };
+  }
+  return undefined;
+}
+
+interface ColumnState {
+  column: CatalogColumn;
+  evidence: SensitivityEvidence[];
+  nonSensitiveEvidence?: SensitivityEvidence;
+  observedValues: number;
+  nerCandidates: string[];
+  coverage: "metadata" | "complete" | "sampled" | "no_values";
+  sampledTarget: number;
+}
+
+/** Sole decision module for local column-level sensitivity assessments. */
+export class SensitivityClassifier {
+  constructor(
+    private readonly values: SensitivityValueSource,
+    private readonly detector?: LocalNerDetector,
+    private readonly options: {
+      queryTimeoutMs?: number;
+      nerConfidenceThreshold?: number;
+      maxNerValuesPerColumn?: number;
+      maxNerCandidatesPerTable?: number;
+      now?: () => number;
+    } = {},
+  ) {}
+
+  async assess(
+    targets: readonly SensitivityTableTarget[],
+    signal: AbortSignal,
+    sharedNerBudget?: SensitivityNerBudget,
+    onActivity?: (message: string) => void | Promise<void>,
+  ): Promise<readonly SensitivityColumnAssessment[]> {
+    const now = this.options.now ?? Date.now;
+    const maxNerValuesPerColumn = boundedCount(this.options.maxNerValuesPerColumn, 8, 8);
+    const states = new Map<string, ColumnState>();
+    for (const target of targets) {
+      for (const column of target.columns) {
+        const nonSensitiveEvidence = nonSensitiveStructuralEvidence(column);
+        const metadataMatch = nonSensitiveEvidence ? undefined : metadataEvidence(column);
+        const binary = !nonSensitiveEvidence && UNSUPPORTED_BINARY_TYPE.test(column.dataType);
+        states.set(column.id, {
+          column,
+          evidence: metadataMatch
+            ? [metadataMatch]
+            : binary
+              ? [{ kind: "type", ruleId: "type.binary_uninspectable" }]
+              : [],
+          ...(nonSensitiveEvidence ? { nonSensitiveEvidence } : {}),
+          observedValues: 0,
+          nerCandidates: [],
+          coverage: nonSensitiveEvidence || metadataMatch || binary ? "metadata" : "no_values",
+          sampledTarget: 0,
+        });
+      }
+    }
+
+    const completeTables = new Set<string>();
+    for (const [phaseIndex, phase] of SENSITIVITY_SAMPLE_PHASES.entries()) {
+      for (let offset = 0; offset < targets.length; offset += MAX_CONCURRENT_TABLE_SCANS) {
+        signal.throwIfAborted();
+        const batchNumber = Math.floor(offset / MAX_CONCURRENT_TABLE_SCANS) + 1;
+        const batchCount = Math.ceil(targets.length / MAX_CONCURRENT_TABLE_SCANS);
+        await onActivity?.(
+          `Scanning source data: pass ${phaseIndex + 1} of ${SENSITIVITY_SAMPLE_PHASES.length}, table batch ${batchNumber} of ${batchCount}.`,
+        );
+        signal.throwIfAborted();
+        const peerController = new AbortController();
+        const scanSignal = AbortSignal.any([signal, peerController.signal]);
+        try {
+          await Promise.all(targets.slice(offset, offset + MAX_CONCURRENT_TABLE_SCANS).map(async (target) => {
+            if (completeTables.has(target.table.id)) return;
+            const columns = target.columns.filter((column) => {
+              const state = states.get(column.id)!;
+              return state.evidence.length === 0 && !state.nonSensitiveEvidence
+                && (!phase.deepTextOnly || DEEP_TEXT_TYPE.test(column.dataType));
+            });
+            if (columns.length === 0) return;
+            const coverage = await this.values.scanTable({
+              ...target,
+              columns,
+              valuesPerColumn: phase.additionalValuesPerColumn,
+              sampleOffset: phase.targetValuesPerColumn - phase.additionalValuesPerColumn,
+              sampleSeed: phase.sampleSeed,
+              queryTimeoutMs: this.options.queryTimeoutMs ?? 5_000,
+              ...(phaseIndex === 0 ? { fullScanThreshold: 1_000 } : {}),
+            }, (batch) => {
+              for (const item of batch) {
+                if (item.value === null) continue;
+                const state = states.get(item.columnId);
+                if (!state || state.evidence.length > 0) continue;
+                state.observedValues += 1;
+                if ((item.characterLength ?? item.value.length) > 500) {
+                  state.evidence.push({ kind: "length", ruleId: "text.over_500_characters" });
+                  continue;
+                }
+                const match = contentEvidence(item.value);
+                if (match) {
+                  state.evidence.push(match);
+                  continue;
+                }
+                if (state.nerCandidates.length < maxNerValuesPerColumn
+                  && !state.nerCandidates.includes(item.value)) {
+                  state.nerCandidates.push(item.value);
+                }
+              }
+            }, scanSignal);
+            for (const column of columns) {
+              const state = states.get(column.id)!;
+              state.sampledTarget = Math.max(state.sampledTarget, phase.targetValuesPerColumn);
+              state.coverage = coverage.kind === "complete"
+                ? "complete"
+                : state.observedValues === 0 ? "no_values" : "sampled";
+            }
+            if (coverage.kind === "complete") completeTables.add(target.table.id);
+          }));
+        } catch (error) {
+          peerController.abort(error);
+          throw error;
+        }
+      }
+    }
+
+    const nerBudget = sharedNerBudget ?? { remainingMs: 10_000 };
+    if (this.detector && (this.detector.isReady?.() ?? true) && !signal.aborted
+      && nerBudget.remainingMs > 0) {
+      const maxCandidates = boundedCount(this.options.maxNerCandidatesPerTable, 2, 1_024);
+      const threshold = this.options.nerConfidenceThreshold ?? 0.8;
+      for (const [targetIndex, target] of targets.entries()) {
+        signal.throwIfAborted();
+        if (nerBudget.remainingMs <= 0) break;
+        const candidates: LocalNerCandidate[] = [];
+        candidateSelection: for (let valueIndex = 0; valueIndex < maxNerValuesPerColumn; valueIndex += 1) {
+          for (const column of target.columns) {
+            const state = states.get(column.id)!;
+            if (state.evidence.length > 0 || state.nonSensitiveEvidence) continue;
+            const text = state.nerCandidates[valueIndex];
+            if (text === undefined) continue;
+            candidates.push({ columnId: column.id, text });
+            if (candidates.length >= maxCandidates) break candidateSelection;
+          }
+        }
+        if (candidates.length === 0) continue;
+        await onActivity?.(
+          `Running local entity detection: table ${targetIndex + 1} of ${targets.length}.`,
+        );
+        signal.throwIfAborted();
+        const startedAt = now();
+        const deadline = startedAt + nerBudget.remainingMs;
+        try {
+          for (let offset = 0; offset < candidates.length; offset += MAX_NER_CANDIDATES_PER_REQUEST) {
+            if (signal.aborted || now() >= deadline) break;
+            try {
+              const detected = await this.detector.detect(
+                candidates.slice(offset, offset + MAX_NER_CANDIDATES_PER_REQUEST),
+                signal,
+                deadline,
+              );
+              for (const item of detected) {
+                const state = states.get(item.columnId);
+                if (!state || state.evidence.length > 0 || !Number.isFinite(item.confidence)
+                  || item.confidence < threshold || item.confidence > 1) continue;
+                const label = normalizedName(item.label).slice(0, 80);
+                if (!label) continue;
+                state.evidence.push({
+                  kind: "ner",
+                  ruleId: "ner.entity",
+                  label,
+                  confidence: item.confidence,
+                });
+              }
+            } catch {
+              // NER is optional: deterministic findings and scan coverage remain authoritative.
+              break;
+            }
+          }
+        } finally {
+          nerBudget.remainingMs = Math.max(0, nerBudget.remainingMs - Math.max(1, now() - startedAt));
+        }
+      }
+    }
+
+    return targets.flatMap((target) => target.columns.map((column) => {
+      const state = states.get(column.id)!;
+      const sensitive = state.evidence.length > 0;
+      const coverage = state.nonSensitiveEvidence
+        ? "metadata"
+        : state.observedValues === 0 && !sensitive ? "no_values" : state.coverage;
+      const coverageEvidence: SensitivityEvidence[] = sensitive
+        ? state.evidence
+        : state.nonSensitiveEvidence
+          ? [state.nonSensitiveEvidence]
+          : [{
+            kind: "coverage",
+            ruleId: coverage === "complete"
+              ? "coverage.complete"
+              : coverage === "no_values"
+                ? "coverage.no_values"
+                : `coverage.sampled_${state.sampledTarget}`,
+          }];
+      return {
+        columnId: column.id,
+        assessment: sensitive ? "sensitive" : "non_sensitive",
+        proposedSensitive: sensitive,
+        evidence: coverageEvidence,
+        observedValues: state.observedValues,
+        coverage,
+      };
+    }));
+  }
+
+  /** Convenience for focused callers and rule-level tests. Production orchestration uses assess(). */
+  async assessTable(
+    target: SensitivityTableTarget,
+    signal: AbortSignal,
+    _retiredRunDeadline?: number,
+    nerBudget?: SensitivityNerBudget,
+  ): Promise<readonly SensitivityColumnAssessment[]> {
+    return await this.assess([target], signal, nerBudget);
+  }
+}

backend/src/catalog/sensitivity-shadow.ts

@@ -0,0 +1,100 @@
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadConfig } from "../config.js";
+import { WorkspaceSecretStore } from "../workspaces/secret-store.js";
+import { PythonLocalNerDetector } from "./local-ner-detector.js";
+import { ConcreteCatalogPostgresAccess } from "./postgres-access.js";
+import { createCatalogRepository } from "./repository.js";
+import {
+  SENSITIVITY_POLICY_VERSION,
+  SensitivityAnalysisService,
+} from "./sensitivity-analysis-service.js";
+import { SensitivityClassifier } from "./sensitivity-classifier.js";
+import { ConcreteSensitivityValueSource } from "./sensitivity-value-source.js";
+
+const WORKSPACE_ID = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u;
+
+async function main(): Promise<void> {
+  const workspaceId = process.argv[2];
+  if (!workspaceId || !WORKSPACE_ID.test(workspaceId)) {
+    process.stderr.write("Usage: sensitivity-shadow <workspace-id>\n");
+    process.exitCode = 2;
+    return;
+  }
+
+  let detector: PythonLocalNerDetector | undefined;
+  let stage = "configuration";
+  try {
+    const config = loadConfig(process.env);
+    stage = "catalog";
+    const repository = createCatalogRepository(config.catalogDatabase);
+    if (!(await repository.available())) throw new Error("catalog unavailable");
+    const database = await repository.getByWorkspace(workspaceId);
+    if (!database) throw new Error("database unavailable");
+    stage = "source";
+    const secretStore = new WorkspaceSecretStore({
+      root: config.workspaceSecretStoreRoot,
+      runtimeRoot: config.workspaceSecretRuntimeRoot,
+      installationId: config.workspaceRegistry.installationId,
+    });
+    const access = new ConcreteCatalogPostgresAccess(secretStore, {
+      connectTimeoutMs: config.workspaceDiagnosticTimeoutMs,
+    });
+    const source = new ConcreteSensitivityValueSource(access, secretStore);
+    if (config.sensitivityNer) {
+      const workerScript = config.sensitivityNer.workerScript
+        ?? fileURLToPath(new URL("../../python/sensitivity_ner_worker.py", import.meta.url));
+      detector = new PythonLocalNerDetector({
+        pythonExecutable: config.sensitivityNer.pythonExecutable,
+        workerScript,
+        modelPath: config.sensitivityNer.modelPath,
+        cwd: dirname(workerScript),
+        threads: config.sensitivityNer.threads,
+      });
+      try {
+        await detector.warmup();
+      } catch {
+        await detector.close();
+        detector = undefined;
+      }
+    }
+    const startedAt = Date.now();
+    stage = "analysis";
+    const suggestions = await new SensitivityAnalysisService(
+      repository,
+      new SensitivityClassifier(source, detector),
+    ).analyze(database.id, "all", [], new AbortController().signal);
+    const assessments = { sensitive: 0, nonSensitive: 0 };
+    const coverage = { metadata: 0, complete: 0, sampled: 0, noValues: 0 };
+    const rules = new Map<string, number>();
+    for (const suggestion of suggestions) {
+      if (suggestion.assessment === "sensitive") assessments.sensitive += 1;
+      else assessments.nonSensitive += 1;
+      if (suggestion.coverage === "no_values") coverage.noValues += 1;
+      else coverage[suggestion.coverage] += 1;
+      for (const evidence of suggestion.evidence) {
+        rules.set(evidence.ruleId, (rules.get(evidence.ruleId) ?? 0) + 1);
+      }
+    }
+    process.stdout.write(`${JSON.stringify({
+      ok: true,
+      policyVersion: SENSITIVITY_POLICY_VERSION,
+      nerEnabled: detector !== undefined,
+      total: suggestions.length,
+      assessments,
+      coverage,
+      rules: Object.fromEntries([...rules].sort(([left], [right]) => left.localeCompare(right))),
+      elapsedMs: Date.now() - startedAt,
+    })}\n`);
+  } catch {
+    process.stdout.write(`${JSON.stringify({
+      ok: false,
+      code: `sensitivity_shadow_${stage}_failed`,
+    })}\n`);
+    process.exitCode = 1;
+  } finally {
+    await detector?.close();
+  }
+}
+
+await main();

backend/src/catalog/sensitivity-value-source.ts

@@ -0,0 +1,291 @@
+import { readFile } from "node:fs/promises";
+import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
+import { CATALOG_SECRET_IDS } from "./secrets.js";
+import type { CatalogPostgresAccess } from "./postgres-access.js";
+import type {
+  SensitivityScanCoverage,
+  SensitivityScanRequest,
+  SensitivityValueObservation,
+  SensitivityValueSource,
+} from "./sensitivity-classifier.js";
+import { CatalogConnectorError, type CatalogColumn } from "./types.js";
+
+const MAX_VALUE_CHARACTERS = 501;
+const MAX_COLUMNS_PER_QUERY = 25;
+const SAMPLE_OVERSCAN_FACTOR = 10;
+
+function quoteIdentifier(identifier: string): string {
+  return `"${identifier.replaceAll('"', '""')}"`;
+}
+
+function chunks<T>(items: readonly T[], size: number): T[][] {
+  const result: T[][] = [];
+  for (let offset = 0; offset < items.length; offset += size) {
+    result.push(items.slice(offset, offset + size));
+  }
+  return result;
+}
+
+function tableReference(request: SensitivityScanRequest): string {
+  return `${quoteIdentifier(request.database.schema)}.${quoteIdentifier(request.table.name)}`;
+}
+
+function samplePercentage(valuesPerColumn: number): number {
+  if (valuesPerColumn <= 300) return 30;
+  if (valuesPerColumn <= 700) return 70;
+  return 100;
+}
+
+function flatValueQuery(
+  request: SensitivityScanRequest,
+  columns: readonly CatalogColumn[],
+  options: { complete: boolean; randomized: boolean },
+): string {
+  const projections = columns.map((column) => quoteIdentifier(column.name)).join(", ");
+  const perColumnLimit = options.complete
+    ? request.fullScanThreshold ?? request.valuesPerColumn
+    : request.valuesPerColumn;
+  const rowLimit = Math.max(perColumnLimit, perColumnLimit * SAMPLE_OVERSCAN_FACTOR);
+  const sample = options.complete
+    ? `SELECT ${projections} FROM ${tableReference(request)}`
+    : [
+      `SELECT ${projections} FROM ${tableReference(request)}`,
+      ...(options.randomized
+        ? [`TABLESAMPLE SYSTEM (${samplePercentage(request.valuesPerColumn)}) REPEATABLE (${request.sampleSeed})`]
+        : []),
+      `LIMIT ${rowLimit} OFFSET ${request.sampleOffset}`,
+    ].join(" ");
+  const values = columns.map((column, index) => {
+    const identifier = quoteIdentifier(column.name);
+    return [
+      `(${index}, LEFT((sampled.${identifier})::text, ${MAX_VALUE_CHARACTERS}),`,
+      `CASE WHEN sampled.${identifier} IS NULL THEN NULL`,
+      `ELSE char_length((sampled.${identifier})::text) END)`,
+    ].join(" ");
+  }).join(", ");
+  return [
+    `WITH sampled AS MATERIALIZED (${sample}),`,
+    "ranked AS (",
+    "SELECT value.__column_index, value.__value, value.__length,",
+    "row_number() OVER (PARTITION BY value.__column_index) AS __rank",
+    "FROM sampled",
+    `CROSS JOIN LATERAL (VALUES ${values}) AS value(__column_index, __value, __length)`,
+    "WHERE value.__value IS NOT NULL",
+    ")",
+    "SELECT __column_index, __value, __length FROM ranked",
+    `WHERE __rank <= ${perColumnLimit}`,
+  ].join(" ");
+}
+
+function observations(
+  columns: readonly CatalogColumn[],
+  rows: readonly Record<string, unknown>[],
+): SensitivityValueObservation[] {
+  return rows.flatMap((row) => {
+    const index = Number(row.__column_index);
+    const column = Number.isSafeInteger(index) && index >= 0 ? columns[index] : undefined;
+    if (!column || row.__value === null || row.__value === undefined) return [];
+    const value = String(row.__value);
+    const parsedLength = row.__length === null || row.__length === undefined
+      ? null
+      : Number(row.__length);
+    return [{
+      columnId: column.id,
+      value,
+      characterLength: parsedLength !== null && Number.isSafeInteger(parsedLength) && parsedLength >= 0
+        ? parsedLength
+        : value.length,
+    }];
+  });
+}
+
+function cancelled(error: unknown): boolean {
+  return Boolean(error && typeof error === "object" && "code" in error && error.code === "57014");
+}
+
+/**
+ * Database-specific sampling adapter. Policy stays in SensitivityClassifier; this module only
+ * produces bounded, normalized non-null observations without persisting or logging values.
+ */
+export class ConcreteSensitivityValueSource implements SensitivityValueSource {
+  constructor(
+    private readonly access: CatalogPostgresAccess,
+    private readonly secretStore?: Pick<WorkspaceSecretStore, "materialize">,
+  ) {}
+
+  async scanTable(
+    request: SensitivityScanRequest,
+    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
+    signal: AbortSignal,
+  ): Promise<SensitivityScanCoverage> {
+    if (request.columns.length === 0) return { kind: "complete", observedValues: 0 };
+    if (request.database.binding.transport === "rest_api") {
+      return await this.scanRest(request, consume, signal);
+    }
+    return await this.scanPostgres(request, consume, signal);
+  }
+
+  private async scanPostgres(
+    request: SensitivityScanRequest,
+    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
+    signal: AbortSignal,
+  ): Promise<SensitivityScanCoverage> {
+    const client = await this.access.connect(request.database, signal);
+    let transactionOpen = false;
+    let savepointSequence = 0;
+    let observedValues = 0;
+    try {
+      signal.throwIfAborted();
+      await client.query("BEGIN TRANSACTION READ ONLY", []);
+      transactionOpen = true;
+      await client.query("SELECT set_config('statement_timeout', $1, true)", [
+        `${Math.max(1, Math.floor(request.queryTimeoutMs))}ms`,
+      ]);
+      const boundedQuery = async (sql: string): Promise<Array<Record<string, unknown>> | undefined> => {
+        signal.throwIfAborted();
+        savepointSequence += 1;
+        const savepoint = `sensitivity_scan_${savepointSequence}`;
+        await client.query(`SAVEPOINT ${savepoint}`, []);
+        try {
+          return (await client.query(sql, [])).rows;
+        } catch (error) {
+          if (!cancelled(error)) throw error;
+          await client.query(`ROLLBACK TO SAVEPOINT ${savepoint}`, []);
+          return undefined;
+        } finally {
+          await client.query(`RELEASE SAVEPOINT ${savepoint}`, []).catch(() => undefined);
+        }
+      };
+
+      let complete = false;
+      if (request.fullScanThreshold !== undefined) {
+        const probe = await boundedQuery(
+          `SELECT 1 AS __present FROM ${tableReference(request)} LIMIT ${request.fullScanThreshold + 1}`,
+        );
+        complete = probe !== undefined && probe.length <= request.fullScanThreshold;
+      }
+      for (const columnChunk of chunks(request.columns, MAX_COLUMNS_PER_QUERY)) {
+        signal.throwIfAborted();
+        let rows = await boundedQuery(flatValueQuery(request, columnChunk, {
+          complete,
+          randomized: !complete,
+        }));
+        if (rows === undefined && complete) {
+          complete = false;
+          rows = await boundedQuery(flatValueQuery(request, columnChunk, {
+            complete: false,
+            randomized: true,
+          }));
+        }
+        if (!complete && (rows === undefined || rows.length === 0)) {
+          rows = await boundedQuery(flatValueQuery(request, columnChunk, {
+            complete: false,
+            randomized: false,
+          }));
+        }
+        if (rows === undefined) throw new CatalogConnectorError("Sensitivity sample query timed out");
+        const batch = observations(columnChunk, rows);
+        observedValues += batch.length;
+        if (batch.length > 0) await consume(batch);
+      }
+      return { kind: complete ? "complete" : "sampled", observedValues };
+    } catch (error) {
+      if (error instanceof CatalogConnectorError) throw error;
+      throw new CatalogConnectorError("Sensitivity source scan failed");
+    } finally {
+      if (transactionOpen) await client.query("ROLLBACK", []).catch(() => undefined);
+      await client.end().catch(() => undefined);
+    }
+  }
+
+  private async scanRest(
+    request: SensitivityScanRequest,
+    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
+    signal: AbortSignal,
+  ): Promise<SensitivityScanCoverage> {
+    if (!this.secretStore) throw new CatalogConnectorError("REST sensitivity scanning is not configured");
+    const auth = request.database.binding.restAuth ?? "bearer";
+    const materialized = this.secretStore.materialize(
+      request.database.workspaceId,
+      auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
+    );
+    let observedValues = 0;
+    try {
+      const headers: Record<string, string> = { "content-type": "application/json" };
+      if (auth !== "none") {
+        const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
+        if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
+        const credential = (await readFile(credentialFile, "utf8")).trim();
+        if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
+        else headers["x-api-key"] = credential;
+      }
+      const baseUrl = request.database.binding.baseUrl?.replace(/\/+$/u, "");
+      if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete");
+      const runQuery = async (sql: string): Promise<Array<Record<string, unknown>> | undefined> => {
+        const timeout = AbortSignal.timeout(Math.max(1, Math.floor(request.queryTimeoutMs)));
+        try {
+          const response = await fetch(`${baseUrl}/rpc/run_query`, {
+            method: "POST",
+            headers,
+            body: JSON.stringify({ query_text: sql }),
+            signal: AbortSignal.any([signal, timeout]),
+          });
+          if (!response.ok) throw new CatalogConnectorError("REST sensitivity source scan failed");
+          const body: unknown = await response.json();
+          if (!Array.isArray(body)
+            || body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
+            throw new CatalogConnectorError("REST sensitivity source response is invalid");
+          }
+          return body as Array<Record<string, unknown>>;
+        } catch (error) {
+          if (signal.aborted) throw error;
+          if (timeout.aborted) return undefined;
+          throw error;
+        }
+      };
+
+      let complete = false;
+      if (request.fullScanThreshold !== undefined) {
+        const probe = await runQuery(
+          `SELECT 1 AS __present FROM ${tableReference(request)} LIMIT ${request.fullScanThreshold + 1}`,
+        );
+        complete = probe !== undefined && probe.length <= request.fullScanThreshold;
+      }
+      let requestCount = request.fullScanThreshold === undefined ? 0 : 1;
+      for (const columnChunk of chunks(request.columns, MAX_COLUMNS_PER_QUERY)) {
+        signal.throwIfAborted();
+        let rows = await runQuery(flatValueQuery(request, columnChunk, {
+          complete,
+          randomized: !complete,
+        }));
+        requestCount += 1;
+        if (rows === undefined && complete) {
+          complete = false;
+          rows = await runQuery(flatValueQuery(request, columnChunk, {
+            complete: false,
+            randomized: true,
+          }));
+          requestCount += 1;
+        }
+        if (!complete && (rows === undefined || rows.length === 0)) {
+          rows = await runQuery(flatValueQuery(request, columnChunk, {
+            complete: false,
+            randomized: false,
+          }));
+          requestCount += 1;
+        }
+        if (rows === undefined) throw new CatalogConnectorError("REST sensitivity sample query timed out");
+        const batch = observations(columnChunk, rows);
+        observedValues += batch.length;
+        if (batch.length > 0) await consume(batch);
+      }
+      // Multiple HTTP requests cannot share a source snapshot, so only one-request reads are complete.
+      return { kind: complete && requestCount === 1 ? "complete" : "sampled", observedValues };
+    } catch (error) {
+      if (error instanceof CatalogConnectorError) throw error;
+      throw new CatalogConnectorError("REST sensitivity source scan failed");
+    } finally {
+      materialized.release();
+    }
+  }
+}

backend/src/catalog/service.ts

@@ -1,5 +1,3 @@
-import { buildInstallationContract } from "../workspaces/contracts.js";
-import { resolveBinding } from "../workspaces/bindings.js";
 import type { WorkspaceRegistry } from "../workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../workspaces/schema.js";
 import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
@@ -45,60 +43,33 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
   secrets: Record<CatalogSecretName, boolean>;
 }
 
-function bindingValue(workspace: WorkspaceDescriptor, values: Record<string, string>, suffix: string) {
-  const variable = buildInstallationContract(workspace).variables.find((entry) => (
-    entry.role === "DWH" && entry.suffix === suffix
-  ));
-  return variable ? values[variable.name] : undefined;
-}
-
-function numeric(value: string | undefined): number | undefined {
-  if (!value) return undefined;
-  const parsed = Number(value);
-  return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65_535 ? parsed : undefined;
-}
-
-function yamlBinding(workspace: WorkspaceDescriptor, secretRoots: readonly string[]): DatabaseBinding {
-  const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
-  const value = (suffix: string) => bindingValue(workspace, effective.values, suffix);
-  return {
-    transport: effective.transport,
-    host: value("HOST"),
-    port: numeric(value("PORT")) ?? workspace.dwh.port,
-    username: value("USER"),
-    baseUrl: value("BASE_URL"),
-    restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
-    restAuth: workspace.diagnostics?.dwh_rest?.auth ?? "bearer",
-    tlsServername: value("TLS_SERVERNAME"),
-    sshHost: value("SSH_HOST"),
-    sshPort: numeric(value("SSH_PORT")),
-    sshUsername: value("SSH_USER"),
-    sshTargetHost: value("SSH_TARGET_HOST"),
-    sshTargetPort: numeric(value("SSH_TARGET_PORT")),
-  };
-}
-
 function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<CatalogSecretName, boolean> {
   return Object.fromEntries(Object.entries(CATALOG_SECRET_IDS).map(([name, id]) => (
     [name, store.has(workspaceId, id)]
   ))) as Record<CatalogSecretName, boolean>;
 }
 
-function workspaceRuntimeState(
+function databaseRuntimeState(
   store: WorkspaceSecretStore,
-  workspace: WorkspaceDescriptor,
-  secretRoots: readonly string[],
+  database: WorkspaceDatabase,
 ): NonNullable<CatalogListItem["runtimeBinding"]> {
-  const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
-  const secretVariables = new Set(requirements.map(({ variable }) => variable));
-  const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
-  const configurationRequired = requirements.some(({ id, required }) => (
-    required && !store.has(workspace.workspace.id, id)
-  )) || effective.missing.some((variable) => !secretVariables.has(variable));
+  const { binding, workspaceId } = database;
+  const configured = (id: string) => store.has(workspaceId, id);
+  const connectionComplete = binding.transport === "postgres_direct"
+    ? Boolean(binding.host && binding.port && binding.username && configured(CATALOG_SECRET_IDS.password))
+    : binding.transport === "rest_api"
+      ? Boolean(binding.baseUrl && (binding.restAuth === "none" || configured(CATALOG_SECRET_IDS.apiKey)))
+      : Boolean(
+        binding.username && binding.sshHost && binding.sshPort && binding.sshUsername
+        && binding.sshTargetHost && binding.sshTargetPort
+        && configured(CATALOG_SECRET_IDS.password)
+        && configured(CATALOG_SECRET_IDS.sshPrivateKey)
+        && configured(CATALOG_SECRET_IDS.sshKnownHosts),
+      );
   return {
-    transport: effective.transport,
-    configurationState: configurationRequired ? "configuration_required" : "ready",
-    sessionTransportSupported: effective.transport !== "ssh_tunnel",
+    transport: binding.transport,
+    configurationState: connectionComplete ? "ready" : "configuration_required",
+    sessionTransportSupported: binding.transport !== "ssh_tunnel",
   };
 }
 
@@ -145,17 +116,18 @@ export class CatalogService {
     const active = await Promise.all(workspaces.map(async (entry) => {
       const database = byWorkspace.get(entry.id);
       const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
-      const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
       const base = database ?? {
         workspaceId: entry.id,
         engine: "postgres" as const,
-        databaseName: workspace.dwh.database,
-        schema: workspace.dwh.schema,
+        databaseName: "",
+        schema: "",
         version: 0,
         createdAt: "",
         updatedAt: "",
-        binding: runtimeDatabaseBinding,
+        binding: { transport: "postgres_direct" as const },
         connectionStatus: "untested" as const,
+        metadataContentRevision: 0,
+        preprocessingStatus: "failed" as const,
       };
       return {
         ...base,
@@ -167,7 +139,7 @@ export class CatalogService {
           blob: entry.revision.blob,
         },
         workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
-        runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
+        runtimeBinding: database ? databaseRuntimeState(this.secretStore, database) : null,
         configured: database !== undefined,
         secrets: secretState(this.secretStore, entry.id),
       };
@@ -195,13 +167,13 @@ export class CatalogService {
   }
 
   async normalizeInput(input: DatabaseConfigurationInput): Promise<DatabaseConfigurationInput> {
-    const workspace = await this.ensureWorkspace(input.workspaceId);
+    await this.ensureWorkspace(input.workspaceId);
     if (input.binding.transport !== "rest_api") return input;
     return {
       ...input,
       binding: {
         ...input.binding,
-        restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
+        restPath: input.binding.restPath ?? "/health",
       },
     };
   }

backend/src/catalog/sync-worker.ts

@@ -39,7 +39,10 @@ function safeFailure(error: unknown): { code: string; message: string } {
     };
   }
   if (error instanceof CatalogConnectorError) {
-    return { code: "schema_introspection_failed", message: "The database schema could not be read safely." };
+    return {
+      code: "schema_introspection_failed",
+      message: "The database schema could not be read. Check the connection and credentials, then try again.",
+    };
   }
   return { code: "schema_sync_failed", message: "Schema synchronization failed." };
 }
@@ -55,6 +58,8 @@ export class CatalogSyncWorker {
     private readonly introspector: CatalogSchemaIntrospector,
     private readonly operations: CatalogOperationCoordinator,
     private readonly timeoutMs: number,
+    private readonly cleanupMemory: (database: WorkspaceDatabase, run: CatalogSyncRun) => Promise<number>
+      = async () => 0,
   ) {}
 
   async initialize(): Promise<void> {
@@ -64,7 +69,9 @@ export class CatalogSyncWorker {
   }
 
   async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise<CatalogSyncRun> {
-    this.assertReady(database);
+    if ((await this.repository.listSyncRuns(database.id, 100)).some(run => run.phase === "memory_cleanup")) {
+      throw new CatalogConflictError("Retry the pending Memory cleanup before starting another synchronization");
+    }
     const uniqueTableIds = [...new Set(tableIds)];
     if (scope === "columns") {
       const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId)));
@@ -107,7 +114,7 @@ export class CatalogSyncWorker {
   async cancel(runId: string): Promise<CatalogSyncRun | undefined> {
     const run = await this.repository.getSyncRun(runId);
     if (!run) return undefined;
-    if (run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
+    if (run.phase === "memory_cleanup" || run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
     await this.repository.requestSyncRunCancellation(runId);
     this.controllers.get(runId)?.abort();
     if (run.state === "queued" || run.state === "awaiting_confirmation") {
@@ -136,6 +143,16 @@ export class CatalogSyncWorker {
     }
     const database = await this.repository.get(previous.databaseId);
     if (!database) return undefined;
+    if (previous.phase === "memory_cleanup") {
+      const release = this.operations.reserve(database.id);
+      try {
+        const queued = await this.repository.updateSyncRun(runId, { state: "queued", cancelRequested: false,
+          errorCode: null, errorMessage: null, finishedAt: null, leaseOwner: null, leaseExpiresAt: null });
+        this.reservations.set(runId, release);
+        this.launch(runId);
+        return queued;
+      } catch (error) { release(); throw error; }
+    }
     return await this.start(database, previous.scope, previous.tableIds);
   }
 
@@ -177,7 +194,10 @@ export class CatalogSyncWorker {
       if (!database || database.version !== claimed.requestedDatabaseVersion) {
         throw new CatalogConflictError("Database binding changed before synchronization started");
       }
-      this.assertReady(database);
+      if (claimed.phase === "memory_cleanup") {
+        await this.finishMemoryCleanup(database, claimed);
+        return;
+      }
       const progress: CatalogSchemaScanProgress = async (phase, counts) => {
         await this.checkCancelled(runId);
         await this.repository.updateSyncRun(runId, {
@@ -229,36 +249,30 @@ export class CatalogSyncWorker {
         claimed.scope,
         claimed.tableIds,
         snapshot,
+        runId,
       );
       if (!applied) throw new CatalogConflictError("Database binding changed before schema changes were applied");
-      await this.repository.updateSyncRun(runId, {
-        state: "succeeded",
-        phase: "completed",
-        counts: applied,
-        finishedAt: new Date().toISOString(),
-        observedSnapshot: null,
-        plannedDiff: null,
-        confirmationToken: null,
-        heartbeatAt: new Date().toISOString(),
-        leaseOwner: null,
-        leaseExpiresAt: null,
-      });
-      await this.repository.appendSyncEvent(runId, "info", "succeeded", "Synchronization completed.", { ...applied });
-      this.release(runId);
+      const cleanupRun = await this.repository.updateSyncRun(runId, { counts: applied });
+      if (!cleanupRun) throw new Error("Synchronization run disappeared");
+      await this.finishMemoryCleanup(database, cleanupRun);
+      /* Completion is recorded only after the durable Memory cleanup succeeds. */
+      return;
     } catch (error) {
       const current = await this.repository.getSyncRun(runId);
       const cancelled = !timedOut && (error instanceof SyncCancelledError || controller.signal.aborted || current?.cancelRequested);
-      const failure = timedOut
+      const failure = current?.phase === "memory_cleanup"
+        ? { code: "memory_cleanup_pending", message: "Catalog synchronized. Memory cleanup is pending; retry this synchronization to complete it." }
+        : timedOut
         ? { code: "schema_sync_timed_out", message: "Schema synchronization timed out." }
         : safeFailure(error);
       await this.repository.updateSyncRun(runId, {
         state: cancelled ? "cancelled" : "failed",
-        phase: "completed",
+        phase: current?.phase === "memory_cleanup" ? "memory_cleanup" : "completed",
         errorCode: cancelled ? null : failure.code,
         errorMessage: cancelled ? null : failure.message,
         finishedAt: new Date().toISOString(),
-        observedSnapshot: null,
-        plannedDiff: null,
+        observedSnapshot: current?.phase === "memory_cleanup" ? current.observedSnapshot : null,
+        plannedDiff: current?.phase === "memory_cleanup" ? current.plannedDiff : null,
         confirmationToken: null,
         leaseOwner: null,
         leaseExpiresAt: null,
@@ -277,10 +291,18 @@ export class CatalogSyncWorker {
     }
   }
 
-  private assertReady(database: WorkspaceDatabase): void {
-    if (database.connectionStatus !== "reachable" || database.testedVersion !== database.version) {
-      throw new CatalogConflictError("Test the current database binding before synchronizing its schema");
-    }
+  private async finishMemoryCleanup(database: WorkspaceDatabase, run: CatalogSyncRun): Promise<void> {
+    if (!run.plannedDiff || run.phase !== "memory_cleanup") throw new Error("Missing committed cleanup context");
+    await this.repository.appendSyncEvent(run.id, "info", "memory_cleanup", "Removing Memory cards with deleted physical dependencies.");
+    const memoryDeleted = await this.cleanupMemory(database, run);
+    const counts = { ...run.counts, memoryDeleted };
+    await this.repository.updateSyncRun(run.id, {
+      state: "succeeded", phase: "completed", counts, finishedAt: new Date().toISOString(),
+      observedSnapshot: null, plannedDiff: null, confirmationToken: null,
+      heartbeatAt: new Date().toISOString(), leaseOwner: null, leaseExpiresAt: null,
+    });
+    await this.repository.appendSyncEvent(run.id, "info", "succeeded", "Synchronization completed.", counts);
+    this.release(run.id);
   }
 
   private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void {

backend/src/catalog/types.ts

@@ -2,6 +2,7 @@ export const DATABASE_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"]
 export type DatabaseTransport = (typeof DATABASE_TRANSPORTS)[number];
 
 export type ConnectionStatus = "untested" | "reachable" | "failed";
+export type CatalogPreprocessingStatus = "running" | "succeeded" | "failed";
 
 export interface DatabaseBinding {
   transport: DatabaseTransport;
@@ -36,8 +37,23 @@ export interface WorkspaceDatabase {
   lastErrorMessage?: string;
   schemaSyncedVersion?: number;
   schemaSyncedAt?: string;
+  metadataContentRevision: number;
+  preprocessingStatus: CatalogPreprocessingStatus;
+  preprocessingInputFingerprint?: string;
+  preprocessedMetadataRevision?: number;
+  preprocessingStartedAt?: string;
+  preprocessingFinishedAt?: string;
+  preprocessingErrorCode?: string;
 }
 
+export type CatalogPreprocessingStartResult =
+  | { kind: "started"; database: WorkspaceDatabase }
+  | { kind: "not_found" | "schema_stale" | "catalog_busy" | "already_running" };
+
+export type CatalogPreprocessingClearResult =
+  | { kind: "cleared"; database: WorkspaceDatabase }
+  | { kind: "not_found" | "already_running" };
+
 export interface CatalogMetrics {
   scope: "global" | "database";
   databaseId: string | null;
@@ -102,6 +118,7 @@ export interface CatalogColumn {
   description: string | null;
   generatedDescription: string | null;
   sensitive: boolean;
+  sensitivityReason: string | null;
   lastSyncedDatabaseVersion: number | null;
   lastSyncedAt: string | null;
   version: number;
@@ -195,7 +212,7 @@ export interface CatalogLogicalRelationshipCandidate {
 
 export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
 export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
-export type CatalogDescriptionTarget = "tables" | "columns";
+export type CatalogDescriptionTarget = "tables" | "columns" | "database" | "database_columns";
 
 export interface CatalogMetadataDeleteCounts {
   tables: number;
@@ -266,18 +283,21 @@ export interface DescriptionGenerationEvent {
   createdAt: string;
 }
 
-export type SensitiveDataSuggestionScope = "all" | "selected_tables" | "selected_columns";
-export type SensitiveDataSuggestionStatus = "running" | "completed" | "failed" | "interrupted";
+export type SensitivityAnalysisScope = "all" | "selected_tables" | "selected_columns";
+export type SensitivityAnalysisStatus = "running" | "completed" | "failed" | "interrupted";
 
-export interface SensitiveDataSuggestionRun {
+export interface SensitivityAnalysisRun {
   id: string;
   databaseId: string;
-  scope: SensitiveDataSuggestionScope;
-  modelId: string;
-  status: SensitiveDataSuggestionStatus;
+  scope: SensitivityAnalysisScope;
+  engine: "llm" | "local";
+  modelId: string | null;
+  policyVersion: string | null;
+  status: SensitivityAnalysisStatus;
   total: number;
   suggestedSensitive: number;
   suggestedNonSensitive: number;
+  unknown: number;
   inputTokens: number;
   cacheReadTokens: number;
   outputTokens: number;
@@ -288,11 +308,12 @@ export interface SensitiveDataSuggestionRun {
   errorSummary: string | null;
 }
 
-export interface SensitiveDataSuggestionRunUpdate {
-  status?: SensitiveDataSuggestionStatus;
+export interface SensitivityAnalysisRunUpdate {
+  status?: SensitivityAnalysisStatus;
   total?: number;
   suggestedSensitive?: number;
   suggestedNonSensitive?: number;
+  unknown?: number;
   finishedAt?: string | null;
   errorSummary?: string | null;
   inputTokens?: number;
@@ -300,7 +321,7 @@ export interface SensitiveDataSuggestionRunUpdate {
   outputTokens?: number;
 }
 
-export interface SensitiveDataSuggestionEvent {
+export interface SensitivityAnalysisEvent {
   runId: string;
   sequence: number;
   level: "info" | "warning" | "error";
@@ -345,7 +366,7 @@ export type CatalogSyncState =
 export type CatalogSyncPhase =
   | "queued" | "connecting" | "scanning_tables" | "scanning_columns"
   | "scanning_relationships" | "planning" | "awaiting_confirmation"
-  | "applying" | "completed";
+  | "applying" | "memory_cleanup" | "completed";
 
 export interface CatalogSchemaDiff {
   deletedTables: string[];
@@ -354,6 +375,7 @@ export interface CatalogSchemaDiff {
 }
 
 export interface CatalogSyncCounts {
+  memoryDeleted?: number;
   tables?: number;
   columns?: number;
   relationships?: number;
@@ -428,6 +450,17 @@ export interface CatalogRepository {
   list(): Promise<WorkspaceDatabase[]>;
   get(id: string): Promise<WorkspaceDatabase | undefined>;
   getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined>;
+  beginPreprocessing(
+    workspaceId: string,
+    inputFingerprint: string,
+  ): Promise<CatalogPreprocessingStartResult>;
+  finishPreprocessing(
+    workspaceId: string,
+    metadataContentRevision: number,
+    inputFingerprint: string,
+    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
+  ): Promise<WorkspaceDatabase | undefined>;
+  clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult>;
   getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined>;
   create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase>;
   update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined>;
@@ -459,6 +492,7 @@ export interface CatalogRepository {
     description: string | null,
     generatedDescription: string | null,
     sensitive?: boolean,
+    sensitivityReason?: string | null,
   ): Promise<CatalogColumn | undefined>;
   consolidateGeneratedDescriptions(
     databaseId: string,
@@ -491,29 +525,29 @@ export interface CatalogRepository {
     runId: string,
     afterSequence?: number,
   ): Promise<DescriptionGenerationEvent[]>;
-  createSensitiveDataSuggestionRun(
+  createSensitivityAnalysisRun(
     databaseId: string,
-    scope: SensitiveDataSuggestionScope,
-    modelId: string,
-  ): Promise<SensitiveDataSuggestionRun>;
-  getSensitiveDataSuggestionRun(runId: string): Promise<SensitiveDataSuggestionRun | undefined>;
-  listSensitiveDataSuggestionRuns(limit?: number): Promise<SensitiveDataSuggestionRun[]>;
-  interruptActiveSensitiveDataSuggestionRuns(
+    scope: SensitivityAnalysisScope,
+    origin: { engine: "llm"; modelId: string } | { engine: "local"; policyVersion: string },
+  ): Promise<SensitivityAnalysisRun>;
+  getSensitivityAnalysisRun(runId: string): Promise<SensitivityAnalysisRun | undefined>;
+  listSensitivityAnalysisRuns(limit?: number): Promise<SensitivityAnalysisRun[]>;
+  interruptActiveSensitivityAnalysisRuns(
     errorSummary: string,
-  ): Promise<SensitiveDataSuggestionRun[]>;
-  updateSensitiveDataSuggestionRun(
+  ): Promise<SensitivityAnalysisRun[]>;
+  updateSensitivityAnalysisRun(
     runId: string,
-    update: SensitiveDataSuggestionRunUpdate,
-  ): Promise<SensitiveDataSuggestionRun | undefined>;
-  appendSensitiveDataSuggestionEvent(
+    update: SensitivityAnalysisRunUpdate,
+  ): Promise<SensitivityAnalysisRun | undefined>;
+  appendSensitivityAnalysisEvent(
     runId: string,
-    level: SensitiveDataSuggestionEvent["level"],
+    level: SensitivityAnalysisEvent["level"],
     message: string,
-  ): Promise<SensitiveDataSuggestionEvent>;
-  listSensitiveDataSuggestionEvents(
+  ): Promise<SensitivityAnalysisEvent>;
+  listSensitivityAnalysisEvents(
     runId: string,
     afterSequence?: number,
-  ): Promise<SensitiveDataSuggestionEvent[]>;
+  ): Promise<SensitivityAnalysisEvent[]>;
   listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]>;
   listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]>;
   getLogicalRelationshipContext(databaseId: string): Promise<CatalogLogicalRelationshipContext | undefined>;
@@ -554,6 +588,7 @@ export interface CatalogRepository {
     scope: CatalogSyncScope,
     tableIds: readonly string[],
     snapshot: ObservedSchemaSnapshot,
+    syncRunId?: string,
   ): Promise<CatalogSyncCounts | undefined>;
   createSyncRun(
     databaseId: string,

backend/src/config.ts

@@ -30,8 +30,18 @@ export interface AppConfig {
   dataRoot?: string;
   ollamaEnsureTimeoutMs: number;
   piManagementTimeoutMs: number;
+  /** Host CLI platform projected into Docker; not the browser or container OS. */
+  hostPlatform?: string;
   secretsFile?: string;
   installationConfigFile?: string;
+  evidenceHostRegistryRoot?: string;
+  modelCatalogFile?: string;
+  sensitivityNer?: {
+    pythonExecutable: string;
+    modelPath: string;
+    workerScript?: string;
+    threads: number;
+  };
   piAuthFile?: string;
   secretFiles: Readonly<Record<string, string | undefined>>;
   modelApiKeyFile?: string;
@@ -50,6 +60,7 @@ export interface AppConfig {
   workspaceSecretRuntimeRoot: string;
   internalQdrantUrl: string;
   internalEmbeddingUrl: string;
+  internalEmbeddingId: string;
   internalEmbeddingModel: string;
   internalEmbeddingDimensions: number;
 }
@@ -189,6 +200,21 @@ function positiveDimension(value: string | undefined, fallback: number): number
   return parsed;
 }
 
+function internalEmbeddingIdentity(
+  identityValue: string | undefined,
+  modelValue: string | undefined,
+): { id: string; model: string } {
+  const id = identityValue ?? "ollama/qwen3-embedding:0.6b";
+  if (!/^ollama\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/.test(id)) {
+    throw new Error("internal embedding identity configuration is invalid");
+  }
+  const model = id.slice(id.indexOf("/") + 1);
+  if (modelValue !== undefined && modelValue !== model) {
+    throw new Error("internal embedding model does not match its canonical identity");
+  }
+  return { id, model };
+}
+
 function catalogDatabase(env: Record<string, string | undefined>): CatalogConnectionConfig | undefined {
   const value = env.THT_CATALOG_DATABASE_URL;
   if (value !== undefined) {
@@ -330,6 +356,42 @@ export function loadConfig(
     || installationConfigFile.includes("\0")
     || !path.isAbsolute(installationConfigFile)
   )) throw new Error("installation configuration is invalid");
+  const modelCatalogFile = env.THT_MODEL_CATALOG_FILE;
+  if (modelCatalogFile !== undefined && (
+    modelCatalogFile.trim() !== modelCatalogFile
+    || modelCatalogFile.length === 0
+    || modelCatalogFile.includes("\0")
+    || !path.isAbsolute(modelCatalogFile)
+  )) throw new Error("runtime model catalog configuration is invalid");
+  const sensitivityNerModelPath = env.THT_SENSITIVITY_NER_MODEL_PATH;
+  const sensitivityNerPython = env.THT_SENSITIVITY_NER_PYTHON;
+  const sensitivityNerWorker = env.THT_SENSITIVITY_NER_WORKER;
+  for (const [value, label] of [
+    [sensitivityNerModelPath, "model path"],
+    [sensitivityNerPython, "Python executable"],
+    [sensitivityNerWorker, "worker path"],
+  ] as const) {
+    if (value !== undefined && (
+      value.length === 0 || value.trim() !== value || value.includes("\0") || !path.isAbsolute(value)
+    )) throw new Error(`sensitivity NER ${label} configuration is invalid`);
+  }
+  if (sensitivityNerModelPath === undefined && (
+    sensitivityNerPython !== undefined
+    || sensitivityNerWorker !== undefined
+    || env.THT_SENSITIVITY_NER_THREADS !== undefined
+  )) throw new Error("sensitivity NER settings require a model path");
+  const sensitivityNerThreads = Number(env.THT_SENSITIVITY_NER_THREADS ?? 2);
+  if (!Number.isSafeInteger(sensitivityNerThreads) || sensitivityNerThreads < 1 || sensitivityNerThreads > 8) {
+    throw new Error("sensitivity NER thread configuration is invalid");
+  }
+  const sensitivityNer = sensitivityNerModelPath === undefined
+    ? undefined
+    : {
+      modelPath: sensitivityNerModelPath,
+      pythonExecutable: sensitivityNerPython ?? "/opt/sensitivity-ner/bin/python",
+      ...(sensitivityNerWorker ? { workerScript: sensitivityNerWorker } : {}),
+      threads: sensitivityNerThreads,
+    };
   const piAuthFile = env.THT_PI_AUTH_FILE;
   if (piAuthFile !== undefined && (
     piAuthFile.trim() !== piAuthFile || piAuthFile.length === 0 || piAuthFile.includes("\0")
@@ -391,6 +453,10 @@ export function loadConfig(
     "internal embedding URL",
     ["embedding", "localhost"],
   );
+  const internalEmbedding = internalEmbeddingIdentity(
+    env.THT_INTERNAL_EMBEDDING_ID,
+    env.THT_INTERNAL_EMBEDDING_MODEL,
+  );
   return {
     host: env.HOST ?? "127.0.0.1",
     port: Number(env.PORT ?? 8787),
@@ -404,15 +470,19 @@ export function loadConfig(
     publicExposure,
     sessionStorage,
     catalogDatabase: catalogDatabase(env),
-    defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING },
+    defaults: { thinking: env.PI_THINKING },
     maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4),
     settingsFile,
     maintenanceFile: env.THT_MAINTENANCE_FILE ?? path.join(path.dirname(settingsFile), "maintenance.json"),
     dataRoot: env.THT_DATA_ROOT,
     ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000),
     piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
+    hostPlatform: env.THT_HOST_PLATFORM,
     secretsFile,
     installationConfigFile,
+    evidenceHostRegistryRoot: env.THT_EVIDENCE_HOST_REGISTRY_ROOT || undefined,
+    modelCatalogFile,
+    sensitivityNer,
     piAuthFile,
     secretFiles,
     modelApiKeyFile,
@@ -425,7 +495,8 @@ export function loadConfig(
     workspaceSecretRuntimeRoot,
     internalQdrantUrl,
     internalEmbeddingUrl,
-    internalEmbeddingModel: env.THT_INTERNAL_EMBEDDING_MODEL ?? "qwen3-embedding:0.6b",
+    internalEmbeddingId: internalEmbedding.id,
+    internalEmbeddingModel: internalEmbedding.model,
     internalEmbeddingDimensions: positiveDimension(env.THT_INTERNAL_EMBEDDING_DIMENSIONS, 1024),
   };
 }

backend/src/config/secret-bundle.ts

@@ -8,7 +8,7 @@ import {
   isUsableAuthenticationSecret,
 } from "../auth/secret-policy.js";
 
-/** Credential names that metadata-generation model entries may reference. */
+/** Credential names that Installation Model Catalog providers may reference. */
 export const METADATA_GENERATION_SECRET_KEYS = Object.freeze([
   "THT_METADATA_API_KEY", "ANTHROPIC_API_KEY", "AZURE_API_KEY", "GEMINI_API_KEY",
   "DEEPSEEK_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ZAI_API_KEY",

backend/src/models/runtime-model-catalog.ts

@@ -0,0 +1,162 @@
+import {
+  closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, type Stats,
+} from "node:fs";
+import { z } from "zod";
+
+const MAX_CATALOG_BYTES = 1024 * 1024;
+const RUNTIME_CATALOG_FILE = "/run/thothii-model-catalog/catalog.json";
+const canonicalId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/);
+const secretBundleKey = /^[A-Z][A-Z0-9_]{0,63}$/;
+
+const endpointSchema = z.object({
+  baseUrl: z.string().url(),
+  apiVersion: z.string().optional(),
+}).strict();
+
+const authenticationSchema = z.object({
+  mode: z.enum(["secret_env", "pi_auth", "none"]),
+  apiKeyEnv: z.string().optional(),
+}).strict();
+
+const runtimeModelSchema = z.object({
+  id: canonicalId,
+  provider: z.string().min(1),
+  model: z.string().min(1),
+  label: z.string().min(1),
+  upstreamModel: z.string().min(1),
+  endpoint: endpointSchema.optional(),
+  authentication: authenticationSchema,
+  sessionAdapter: z.object({ mode: z.enum(["pi_builtin", "openai_compatible"]) }).strict().optional(),
+  metadataAdapter: z.object({ litellmProvider: z.string().min(1) }).strict().optional(),
+  session: z.object({
+    reasoning: z.boolean(),
+    input: z.array(z.string()).optional(),
+    cost: z.object({
+      input: z.number(), output: z.number(), cacheRead: z.number(), cacheWrite: z.number(),
+    }).strict().optional(),
+    contextWindow: z.number().int().positive().optional(),
+    maxTokens: z.number().int().positive().optional(),
+    compatibility: z.object({
+      supportsDeveloperRole: z.boolean(),
+      supportsReasoningEffort: z.boolean(),
+      supportsStore: z.boolean(),
+      maxTokensField: z.string().optional(),
+      thinkingFormat: z.enum(["qwen", "qwen-chat-template"]).optional(),
+    }).strict().optional(),
+  }).strict().refine((session) => !session.compatibility?.thinkingFormat || session.reasoning, {
+    message: "thinkingFormat requires reasoning: true",
+  }).optional(),
+  metadataGeneration: z.object({ disableThinking: z.boolean() }).strict().optional(),
+}).strict();
+
+const catalogSchema = z.object({
+  schemaVersion: z.literal(2),
+  defaultInteraction: canonicalId,
+  embedding: z.object({ id: canonicalId, dimensions: z.number().int().positive() }).strict(),
+  models: z.array(runtimeModelSchema).max(64),
+}).strict();
+
+export type RuntimeModel = z.infer<typeof runtimeModelSchema>;
+
+export interface RuntimeModelCatalog {
+  readonly defaultInteraction: string | null;
+  readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
+  sessionModels(): readonly RuntimeModel[];
+  metadataModels(): readonly RuntimeModel[];
+  hasSession(id: string): boolean;
+}
+
+class RestartLoadedRuntimeModelCatalog implements RuntimeModelCatalog {
+  readonly defaultInteraction: string | null;
+  readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
+  readonly #sessions: readonly RuntimeModel[];
+  readonly #metadata: readonly RuntimeModel[];
+  readonly #sessionIds: ReadonlySet<string>;
+
+  constructor(catalog?: z.infer<typeof catalogSchema>) {
+    this.defaultInteraction = catalog?.defaultInteraction ?? null;
+    this.embedding = catalog ? Object.freeze({ ...catalog.embedding }) : null;
+    // One operational list. Session-only installations can still run Core, but once Admin
+    // LLM models are configured every selectable model must support both adapters.
+    const hasMetadata = catalog?.models.some((model) => model.metadataGeneration !== undefined);
+    this.#sessions = Object.freeze((catalog?.models ?? []).filter((model) =>
+      model.session !== undefined && (!hasMetadata || model.metadataGeneration !== undefined)));
+    this.#metadata = Object.freeze(this.#sessions.filter((model) => model.metadataGeneration !== undefined));
+    this.#sessionIds = new Set(this.#sessions.map((model) => model.id));
+  }
+
+  sessionModels(): readonly RuntimeModel[] { return this.#sessions.map((model) => ({ ...model })); }
+  metadataModels(): readonly RuntimeModel[] { return this.#metadata.map((model) => ({ ...model })); }
+  hasSession(id: string): boolean { return this.#sessionIds.has(id); }
+}
+
+function protectedCatalogStat(file: string, info: Stats): boolean {
+  const mode = info.mode & 0o777;
+  if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
+    || info.size < 1 || info.size > MAX_CATALOG_BYTES) return false;
+  if (file === RUNTIME_CATALOG_FILE && info.uid === 0 && (mode === 0o444 || mode === 0o644)) return true;
+  return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600 || mode === 0o644);
+}
+
+function readProtectedCatalog(file: string): unknown {
+  let descriptor: number | undefined;
+  try {
+    const before = lstatSync(file);
+    if (!protectedCatalogStat(file, before)) throw new Error("runtime model catalog is unavailable");
+    descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
+    const opened = fstatSync(descriptor);
+    if (!protectedCatalogStat(file, opened)
+      || before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("runtime model catalog is unavailable");
+    const source = readFileSync(descriptor, "utf8");
+    const after = fstatSync(descriptor);
+    const current = lstatSync(file);
+    if (!protectedCatalogStat(file, after) || !protectedCatalogStat(file, current)
+      || opened.dev !== after.dev || opened.ino !== after.ino
+      || opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("runtime model catalog is unavailable");
+    return JSON.parse(source);
+  } catch {
+    throw new Error("runtime model catalog is unavailable");
+  } finally {
+    if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized above */ }
+  }
+}
+
+export function loadRuntimeModelCatalog(file?: string): RuntimeModelCatalog {
+  if (!file) return new RestartLoadedRuntimeModelCatalog();
+  const parsed = catalogSchema.safeParse(readProtectedCatalog(file));
+  if (!parsed.success) throw new Error("runtime model catalog is invalid");
+  if (parsed.data.models.some((model) => !validRuntimeModel(model))) {
+    throw new Error("runtime model catalog is invalid");
+  }
+  const ids = new Set(parsed.data.models.map((model) => model.id));
+  if (ids.size !== parsed.data.models.length) throw new Error("runtime model catalog contains duplicate models");
+  const sessions = parsed.data.models.filter((model) => model.session !== undefined).map((model) => model.id);
+  const metadata = parsed.data.models.filter((model) => model.metadataGeneration !== undefined).map((model) => model.id);
+  if (!sessions.includes(parsed.data.defaultInteraction)
+    || (metadata.length > 0 && !metadata.includes(parsed.data.defaultInteraction))) {
+    throw new Error("runtime model catalog interaction default is invalid");
+  }
+  return new RestartLoadedRuntimeModelCatalog(parsed.data);
+}
+
+function validRuntimeModel(model: RuntimeModel): boolean {
+  if ((model.session !== undefined) !== (model.sessionAdapter !== undefined)) return false;
+  if ((model.metadataGeneration !== undefined) !== (model.metadataAdapter !== undefined)) return false;
+  switch (model.authentication.mode) {
+    case "secret_env":
+      return model.authentication.apiKeyEnv !== undefined
+        && secretBundleKey.test(model.authentication.apiKeyEnv);
+    case "pi_auth":
+      return model.authentication.apiKeyEnv === undefined
+        && model.metadataGeneration === undefined
+        && model.sessionAdapter?.mode === "pi_builtin";
+    case "none":
+      return model.authentication.apiKeyEnv === undefined && model.endpoint !== undefined;
+  }
+}
+
+export function splitCanonicalModelId(id: string): { provider: string; model: string } {
+  const slash = id.indexOf("/");
+  if (slash <= 0 || slash === id.length - 1) throw new Error("model identity is invalid");
+  return { provider: id.slice(0, slash), model: id.slice(slash + 1) };
+}

backend/src/operator-command.ts

@@ -8,18 +8,20 @@ import { join } from "node:path";
 import { loadConfig, type AppConfig } from "./config.js";
 import { rolesToPermissions } from "./auth/config.js";
 import type { PrincipalContext } from "./auth/principal.js";
-import { createPiModelLister } from "./pi/list-models.js";
 import { createPiManagement } from "./pi/management.js";
+import { loadRuntimeModelCatalog } from "./models/runtime-model-catalog.js";
 import { effectiveSettings } from "./routes/settings.js";
 import { MaintenanceBarrier } from "./runtime/maintenance-gate.js";
 import { loadSettings } from "./settings/settings-store.js";
 import { ThtRunner, type SessionRow } from "./tht/tht-runner.js";
 import { WorkspaceRegistry } from "./workspaces/registry.js";
 import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
+import { createCatalogRepository } from "./catalog/repository.js";
+import type { CatalogRepository } from "./catalog/types.js";
 
 type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status"
   | "session-inventory" | "workflow-doctor" | "workspace-integrity"
-  | "pi-options" | "pi-test" | "effective-settings";
+  | "pi-test" | "effective-settings";
 
 const lifecyclePrincipal: PrincipalContext = {
   issuer: "tht-operator-command",
@@ -30,7 +32,7 @@ const lifecyclePrincipal: PrincipalContext = {
   isAdmin: true,
 };
 
-function operatorRunner(config: AppConfig): ThtRunner {
+function operatorRunner(config: AppConfig, catalogRepository: CatalogRepository): ThtRunner {
   const workspaceSecretStore = new WorkspaceSecretStore({
     root: config.workspaceSecretStoreRoot,
     runtimeRoot: config.workspaceSecretRuntimeRoot,
@@ -46,6 +48,7 @@ function operatorRunner(config: AppConfig): ThtRunner {
     secretsFile: config.secretsFile,
     secretFiles: config.secretFiles,
     workspaceSecretStore,
+    catalogRepository,
     semanticRuntime: {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
@@ -55,36 +58,55 @@ function operatorRunner(config: AppConfig): ThtRunner {
   }).withPrincipal(lifecyclePrincipal);
 }
 
+async function withOperatorRunner<T>(
+  config: AppConfig,
+  operation: (runner: ThtRunner) => Promise<T>,
+): Promise<T> {
+  const catalogRepository = createCatalogRepository(config.catalogDatabase);
+  try {
+    return await operation(operatorRunner(config, catalogRepository));
+  } finally {
+    await catalogRepository.close?.();
+  }
+}
+
 async function sessionInventory(config: AppConfig): Promise<Array<Pick<SessionRow, "status" | "archived">>> {
   const registry = new WorkspaceRegistry(config.workspaceRegistry);
   const revisions = await registry.listRetainedSnapshots();
-  const runner = operatorRunner(config);
-  const sessions = new Map<string, SessionRow>();
-  for (const revision of revisions) {
-    for (const session of await runner.sessionList(revision.snapshotPath)) sessions.set(session.id, session);
-  }
-  return [...sessions.values()].map(({ status, archived }) => ({ status, archived: archived === true }));
+  return await withOperatorRunner(config, async (runner) => {
+    const sessions = new Map<string, SessionRow>();
+    for (const revision of revisions) {
+      for (const session of await runner.sessionList(revision.snapshotPath)) sessions.set(session.id, session);
+    }
+    return [...sessions.values()].map(({ status, archived }) => ({ status, archived: archived === true }));
+  });
 }
 
 async function workflowDiagnostics(config: AppConfig): Promise<{ ready: true; workspaces: number }> {
   const registry = new WorkspaceRegistry(config.workspaceRegistry);
   const revisions = await registry.listRetainedSnapshots();
   if (revisions.length === 0) throw new Error("workflow diagnostics unavailable");
-  const runner = operatorRunner(config);
-  for (const revision of revisions) {
-    const result = await runner.run(["doctor", "--json"], revision.snapshotPath);
-    let payload: unknown;
-    try {
-      payload = JSON.parse(result.stdout);
-    } catch {
-      throw new Error("workflow diagnostics failed");
+  return await withOperatorRunner(config, async (runner) => {
+    for (const revision of revisions) {
+      const runtime = await runner.acquireWorkspaceRuntime(revision.snapshotPath);
+      try {
+        const result = await runner.run(["doctor", "--json"], runtime.path);
+        let payload: unknown;
+        try {
+          payload = JSON.parse(result.stdout);
+        } catch {
+          throw new Error("workflow diagnostics failed");
+        }
+        if (
+          result.code !== 0 || !payload || typeof payload !== "object"
+          || (payload as { ok?: unknown }).ok !== true
+        ) throw new Error("workflow diagnostics failed");
+      } finally {
+        runtime.release();
+      }
     }
-    if (
-      result.code !== 0 || !payload || typeof payload !== "object"
-      || (payload as { ok?: unknown }).ok !== true
-    ) throw new Error("workflow diagnostics failed");
-  }
-  return { ready: true, workspaces: revisions.length };
+    return { ready: true, workspaces: revisions.length };
+  });
 }
 
 async function workspaceIntegrity(config: AppConfig): Promise<{
@@ -110,9 +132,11 @@ export async function runOperatorAction(
   if (action === "session-inventory") return await sessionInventory(config);
   if (action === "workflow-doctor") return await workflowDiagnostics(config);
   if (action === "workspace-integrity") return await workspaceIntegrity(config);
-  if (action === "effective-settings") return effectiveSettings(config, loadSettings(config));
-  const service = createPiManagement(config, { listModels: createPiModelLister(config) });
-  if (action === "pi-options") return await service.options();
+  const modelCatalog = loadRuntimeModelCatalog(config.modelCatalogFile);
+  if (action === "effective-settings") {
+    return effectiveSettings(config, loadSettings(config), modelCatalog);
+  }
+  const service = createPiManagement(config, { modelCatalog });
   if (action === "pi-test") return await service.test();
   throw new Error("unsupported operator action");
 }
@@ -121,7 +145,7 @@ async function main(): Promise<void> {
   const action = process.argv[2] as OperatorAction | undefined;
   if (!action || ![
     "maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory",
-    "workflow-doctor", "workspace-integrity", "pi-options", "pi-test", "effective-settings",
+    "workflow-doctor", "workspace-integrity", "pi-test", "effective-settings",
   ].includes(action)) throw new Error("invalid operator action");
   const result = await runOperatorAction(action, loadConfig(process.env));
   process.stdout.write(`${JSON.stringify(result)}\n`);

backend/src/pi/list-models.ts

@@ -10,6 +10,8 @@ import {
   readConfiguredPiAgentFile,
   validateDeclarativePiConfig,
 } from "./managed-config.js";
+import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
+import { secretValue } from "../config/secret-bundle.js";
 
 export interface PiModel {
   provider: string;
@@ -18,6 +20,8 @@ export interface PiModel {
   reasoning: boolean;
 }
 
+export type ListModelsFn = () => Promise<PiModel[]>;
+
 interface Opts {
   spawnFn?: (
     command: string,
@@ -28,6 +32,7 @@ interface Opts {
   nowMs?: () => number;
   loadEnabledModels?: () => PiEnabledModelsResult;
   readModelsStore?: () => string | undefined;
+  modelCatalog?: RuntimeModelCatalog;
   warn?: (message: string) => void;
 }
 
@@ -36,7 +41,7 @@ interface Opts {
  * configured) via an ephemeral `pi --mode rpc` process. Result is cached for
  * `ttlMs`. The returned function rejects on timeout/error; callers degrade.
  */
-export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): () => Promise<PiModel[]> {
+export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModelsFn {
   const ttlMs = opts.ttlMs ?? 60_000;
   const now = opts.nowMs ?? (() => Date.now());
   const spawnFn = opts.spawnFn ?? nodeSpawn;
@@ -60,6 +65,14 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): () => Prom
     }
 
     const env = buildPiChildEnv({});
+    // Pi's availability enumeration also needs the catalog-owned credentials for built-in
+    // providers. It must keep working after their obsolete Pi auth entries are removed.
+    for (const model of opts.modelCatalog?.sessionModels() ?? []) {
+      const name = model.authentication.mode === "secret_env" ? model.authentication.apiKeyEnv : undefined;
+      if (!name) continue;
+      const value = secretValue(cfg, name);
+      if (value) env[name] = value;
+    }
     delete env.THT_DATA_ROOT;
     if (cfg.dataRoot !== undefined) env.THT_DATA_ROOT = cfg.dataRoot;
     const child = spawnFn(cfg.piBin, ["--mode", "rpc"], { cwd: cfg.harnessDir, env });
@@ -80,9 +93,23 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): () => Prom
       const byCompositeId = new Map(
         available.map((model) => [`${model.provider}/${model.id}`, model]),
       );
+      const catalogByPiId = new Map(
+        (opts.modelCatalog?.sessionModels() ?? []).map((model) => [
+          `${model.provider}/${model.upstreamModel}`,
+          model,
+        ]),
+      );
       const models = enabled.ids.flatMap((id) => {
         const model = byCompositeId.get(id);
-        return model ? [model] : [];
+        if (!model) return [];
+        const catalogModel = catalogByPiId.get(id);
+        return [catalogModel ? {
+          ...model,
+          provider: catalogModel.provider,
+          id: catalogModel.model,
+          name: catalogModel.label,
+          reasoning: catalogModel.session?.reasoning ?? model.reasoning,
+        } : model];
       });
       if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available");
       cache = { at: now(), models };

backend/src/pi/managed-config.ts

@@ -138,7 +138,7 @@ export interface PiRuntimeAgentSnapshot {
  * Bind a session Pi process to the exact managed auth/model bytes validated at spawn time.
  * Other agent resources remain live through symlinks, while session storage stays persistent.
  */
-export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot {
+export function createPiRuntimeAgentSnapshot(options: { excludeAuthProvider?: string } = {}): PiRuntimeAgentSnapshot {
   const sourceAgentDir = configuredPiAgentDir();
   const auth = readPiAgentFile(sourceAgentDir, "auth.json", true);
   const models = readPiAgentFile(sourceAgentDir, "models.json", true);
@@ -165,7 +165,18 @@ export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot {
       );
     }
     if (auth !== undefined) {
-      writeFileSync(join(snapshotDir, "auth.json"), auth, { flag: "wx", mode: 0o600 });
+      let effectiveAuth = auth;
+      if (options.excludeAuthProvider) {
+        const parsed = parsePiConfigJson(auth);
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new PiManagedConfigError();
+        const provider = options.excludeAuthProvider.trim().toLowerCase();
+        effectiveAuth = JSON.stringify(Object.fromEntries(
+          Object.entries(parsed).filter(([key]) => key.trim().toLowerCase() !== provider),
+        ));
+      }
+      // secret_env is authoritative for this provider. Keep the operator's auth file intact,
+      // but do not let an old Pi credential override the shared bundle inside this child.
+      writeFileSync(join(snapshotDir, "auth.json"), effectiveAuth, { flag: "wx", mode: 0o600 });
     }
     if (models !== undefined) {
       writeFileSync(join(snapshotDir, "models.json"), models, { flag: "wx", mode: 0o600 });

backend/src/pi/management.ts

@@ -2,12 +2,11 @@ import { execFile as nodeExecFile } from "node:child_process";
 import { promisify } from "node:util";
 import type { AppConfig } from "../config.js";
 import { secretValue } from "../config/secret-bundle.js";
+import { loadSettings, type Settings } from "../settings/settings-store.js";
 import {
-  loadSettings,
-  saveSettings,
-  type Settings,
-} from "../settings/settings-store.js";
-import type { PiModel } from "./list-models.js";
+  splitCanonicalModelId,
+  type RuntimeModelCatalog,
+} from "../models/runtime-model-catalog.js";
 import {
   configuredPiProviderApiKey,
   PI_MANAGED_CONFIG_ERROR_MESSAGE,
@@ -37,6 +36,7 @@ export interface PiInstallationConfig {
 }
 
 export interface PiStatus {
+  hostPlatform: "linux" | "macos" | "windows";
   version?: string;
   ready: boolean;
   credentials: PiCredentialStatus;
@@ -45,13 +45,6 @@ export interface PiStatus {
   message?: string;
 }
 
-export interface PiOptions {
-  providers: string[];
-  models: Array<{ provider: string; id: string }>;
-  reasoning: PiReasoning[];
-  checkedAt: string;
-}
-
 export interface PiTestResult {
   ready: boolean;
   checkedAt: string;
@@ -76,15 +69,13 @@ export type PiExecFile = (
 
 export interface PiManagementService {
   status(): Promise<PiStatus>;
-  options(): Promise<PiOptions>;
-  configure(value: PiInstallationConfig): Promise<PiInstallationConfig & { updatedAt: string }>;
   test(): Promise<PiTestResult>;
   logs(): Promise<PiLogs>;
 }
 
 export class PiManagementError extends Error {
   constructor(
-    public readonly code: "pi_management_invalid_config" | "pi_management_unavailable" | "pi_management_write_failed",
+    public readonly code: "pi_management_unavailable",
     message: string,
   ) {
     super(message);
@@ -93,16 +84,18 @@ export class PiManagementError extends Error {
 
 interface PiManagementDeps {
   execute?: PiExecFile;
-  listModels: () => Promise<PiModel[]>;
+  modelCatalog: RuntimeModelCatalog;
   smokeProvider?: PiProviderSmoke;
   readSettings?: () => Settings;
-  saveSettings?: (settings: Settings) => Settings;
   readLogs?: () => string | Promise<string>;
   credentialStatus?: (provider: string | undefined) => PiCredentialStatus;
   now?: () => Date;
 }
 
 export function createPiManagement(config: AppConfig, deps: PiManagementDeps): PiManagementService {
+  const platform = config.hostPlatform ?? process.platform;
+  const hostPlatform = platform === "darwin" ? "macos"
+    : platform === "windows" || platform === "win32" ? "windows" : "linux";
   const now = deps.now ?? (() => new Date());
   const diagnostics: string[] = [];
   const addDiagnostic = (message: string): void => {
@@ -111,54 +104,36 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
   };
   const execute = deps.execute ?? defaultExecFile;
   const readSettings = deps.readSettings ?? (() => loadSettings(config));
-  const persistSettings = deps.saveSettings ?? ((settings) => saveSettings(config, settings));
   const readLogs = deps.readLogs ?? (() => diagnostics.join("\n"));
-  const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config);
+  const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config, {
+    modelCatalog: deps.modelCatalog,
+  });
   const credentialStatus = deps.credentialStatus ?? ((provider: string | undefined) => {
     try {
+      const model = deps.modelCatalog.defaultInteraction
+        ? deps.modelCatalog.sessionModels().find((entry) => entry.id === deps.modelCatalog.defaultInteraction)
+        : undefined;
+      const credentialName = model?.authentication.mode === "secret_env"
+        ? model.authentication.apiKeyEnv
+        : undefined;
+      const configuredApiKey = credentialName ? `$${credentialName}` : configuredPiProviderApiKey(
+        readConfiguredPiAgentFile("models.json", true),
+        provider,
+      );
       return piProviderCredentialStatus({
         provider,
-        authProviders: loadPiAuthProviders(),
-        resolveCredentialValue: () => secretValue(config, "THT_MODEL_API_KEY"),
-        credentialFile: config.modelApiKeyFile,
-        configuredApiKey: configuredPiProviderApiKey(
-          readConfiguredPiAgentFile("models.json", true),
-          provider,
-        ),
+        authProviders: credentialName ? new Set() : loadPiAuthProviders(),
+        resolveCredentialValue: () => credentialName
+          ? secretValue(config, credentialName)
+          : config.modelCatalogFile ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
+        credentialFile: credentialName ? undefined : config.modelApiKeyFile,
+        configuredApiKey,
       });
     } catch {
       return "missing";
     }
   });
 
-  const closedOptions = async (): Promise<Omit<PiOptions, "checkedAt">> => {
-    let listed: PiModel[];
-    try {
-      listed = await deps.listModels();
-    } catch (error) {
-      if (isPiManagedConfigError(error)) {
-        throw new PiManagementError("pi_management_unavailable", PI_MANAGED_CONFIG_ERROR_MESSAGE);
-      }
-      throw new PiManagementError("pi_management_unavailable", "Pi model choices are unavailable");
-    }
-    const models: Array<{ provider: string; id: string }> = [];
-    const providers: string[] = [];
-    const seenModels = new Set<string>();
-    const seenProviders = new Set<string>();
-    for (const model of listed) {
-      if (!isChoice(model?.provider) || !isChoice(model?.id)) continue;
-      const key = `${model.provider}\u0000${model.id}`;
-      if (seenModels.has(key)) continue;
-      seenModels.add(key);
-      models.push({ provider: model.provider, id: model.id });
-      if (!seenProviders.has(model.provider)) {
-        seenProviders.add(model.provider);
-        providers.push(model.provider);
-      }
-    }
-    return { providers, models, reasoning: [...REASONING_CHOICES] };
-  };
-
   const version = async (timeoutMs = config.piManagementTimeoutMs): Promise<string> => {
     let output: { stdout: string; stderr: string };
     try {
@@ -180,12 +155,12 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
 
   const installationConfig = (): PiInstallationConfig => {
     const settings = readSettings();
-    const provider = config.defaults.provider ?? settings.provider;
-    const model = config.defaults.model ?? settings.model;
     const reasoning = config.defaults.thinking ?? settings.thinking;
+    const selected = deps.modelCatalog.defaultInteraction
+      ? splitCanonicalModelId(deps.modelCatalog.defaultInteraction)
+      : undefined;
     return {
-      ...(isChoice(provider) ? { provider } : {}),
-      ...(isChoice(model) ? { model } : {}),
+      ...(selected ? selected : {}),
       ...(isReasoning(reasoning) ? { reasoning } : {}),
     };
   };
@@ -198,36 +173,14 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
       try {
         const currentVersion = await version();
         addDiagnostic("Pi version probe succeeded");
-        return { version: currentVersion, ready: true, credentials, config: current, checkedAt };
+        return { hostPlatform, version: currentVersion, ready: true, credentials, config: current, checkedAt };
       } catch (error) {
         const message = stableMessage(error, "Pi runtime is unavailable");
         addDiagnostic(message);
-        return { ready: false, credentials, config: current, checkedAt, message };
+        return { hostPlatform, ready: false, credentials, config: current, checkedAt, message };
       }
     },
 
-    async options(): Promise<PiOptions> {
-      const choices = await closedOptions();
-      return { ...choices, checkedAt: now().toISOString() };
-    },
-
-    async configure(value: PiInstallationConfig): Promise<PiInstallationConfig & { updatedAt: string }> {
-      if (!isInstallationConfig(value)) {
-        throw new PiManagementError("pi_management_invalid_config", "Pi installation configuration is invalid");
-      }
-      const choices = await closedOptions();
-      if (!choices.models.some((model) => model.provider === value.provider && model.id === value.model)) {
-        throw new PiManagementError("pi_management_invalid_config", "Pi provider and model must be selected from available choices");
-      }
-      try {
-        persistSettings({ ...readSettings(), provider: value.provider, model: value.model, thinking: value.reasoning });
-      } catch {
-        throw new PiManagementError("pi_management_write_failed", "Pi installation configuration could not be saved");
-      }
-      addDiagnostic("Pi installation defaults updated");
-      return { ...value, updatedAt: now().toISOString() };
-    },
-
     async test(): Promise<PiTestResult> {
       const checkedAt = now().toISOString();
       const deadline = Date.now() + config.piManagementTimeoutMs;
@@ -293,24 +246,10 @@ async function defaultExecFile(command: string, args: string[], options: PiExecF
   return { stdout: String(result.stdout), stderr: String(result.stderr) };
 }
 
-function isChoice(value: unknown): value is string {
-  return typeof value === "string" && value.length > 0 && value.length <= 128 && value.trim() === value
-    && /^[A-Za-z0-9][A-Za-z0-9._/-]*$/u.test(value);
-}
-
 function isReasoning(value: unknown): value is PiReasoning {
   return typeof value === "string" && (REASONING_CHOICES as readonly string[]).includes(value);
 }
 
-function isInstallationConfig(value: unknown): value is Required<PiInstallationConfig> {
-  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
-  const candidate = value as Record<string, unknown>;
-  if (Object.keys(candidate).length !== 3 || Object.keys(candidate).some((key) => !["provider", "model", "reasoning"].includes(key))) {
-    return false;
-  }
-  return isChoice(candidate.provider) && isChoice(candidate.model) && isReasoning(candidate.reasoning);
-}
-
 function isTimeout(error: unknown): boolean {
   return Boolean(
     error && typeof error === "object" && (

backend/src/pi/pi-process-manager.ts

@@ -11,6 +11,10 @@ import {
   configuredPiProviderApiKey,
   createPiRuntimeAgentSnapshot,
 } from "./managed-config.js";
+import {
+  loadRuntimeModelCatalog,
+  type RuntimeModelCatalog,
+} from "../models/runtime-model-catalog.js";
 
 export interface SessionRuntime {
   rpc: RpcClient;
@@ -21,6 +25,7 @@ export interface SessionRuntime {
 }
 
 export interface RuntimeOptions {
+  interactionLanguage?: string | null;
   provider?: string;
   model?: string;
   thinking?: string;
@@ -42,23 +47,33 @@ export class PiProcessManager {
   private runtimes = new Map<string, SessionRuntime>();
   private agentSnapshotCleanups = new WeakMap<ChildProcessWithoutNullStreams, () => void>();
   private spawnFn: (
-    sessionId: string, author: string, provider: string | undefined, principal?: PrincipalContext,
-    runtimeConfigPath?: string,
+    sessionId: string, author: string, provider: string | undefined, model: string | undefined,
+    principal?: PrincipalContext, runtimeConfigPath?: string,
+    interactionLanguage?: string | null,
   ) => ChildProcessWithoutNullStreams;
   private loadAuthProviders: (agentDir: string) => ReadonlySet<string>;
+  private modelCatalog: RuntimeModelCatalog;
+  private modelCatalogConfigured: boolean;
 
   constructor(
     private cfg: AppConfig,
-    opts?: { spawnFn?: SpawnFn; authProviders?: (agentDir: string) => ReadonlySet<string> },
+    opts?: {
+      spawnFn?: SpawnFn;
+      authProviders?: (agentDir: string) => ReadonlySet<string>;
+      modelCatalog?: RuntimeModelCatalog;
+    },
   ) {
+    this.modelCatalog = opts?.modelCatalog ?? loadRuntimeModelCatalog(cfg.modelCatalogFile);
+    this.modelCatalogConfigured = cfg.modelCatalogFile !== undefined
+      || this.modelCatalog.defaultInteraction !== null;
     this.loadAuthProviders = opts?.authProviders
       ?? ((agentDir) => loadPiAuthProviders({ agentDir }));
     if (opts?.spawnFn) {
-      this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) =>
-        this.spawnPi(opts.spawnFn!, sessionId, author, provider, principal, runtimeConfigPath);
+      this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath, language) =>
+        this.spawnPi(opts.spawnFn!, sessionId, author, provider, model, principal, runtimeConfigPath, language);
     } else {
-      this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) =>
-        this.spawnPi(nodeSpawn, sessionId, author, provider, principal, runtimeConfigPath);
+      this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath, language) =>
+        this.spawnPi(nodeSpawn, sessionId, author, provider, model, principal, runtimeConfigPath, language);
     }
   }
 
@@ -71,23 +86,36 @@ export class PiProcessManager {
 
   private spawnPi(
     spawnFn: SpawnFn, sessionId: string, author: string, provider: string | undefined,
-    principal?: PrincipalContext, runtimeConfigPath?: string,
+    model: string | undefined, principal?: PrincipalContext, runtimeConfigPath?: string,
+    interactionLanguage?: string | null,
   ): ChildProcessWithoutNullStreams {
     // This is the final shared boundary for createFor(), spawnFor(), and resume(). Validate
     // before auth-provider inspection, then make Pi consume the exact copied bytes rather than
     // reopening mutable mounted auth/models files after this check.
-    const agent = createPiRuntimeAgentSnapshot();
+    const catalogModel = provider && model
+      ? this.modelCatalog.sessionModels().find((entry) => entry.provider === provider && entry.model === model)
+      : undefined;
+    const credentialName = catalogModel?.authentication.mode === "secret_env"
+      ? catalogModel.authentication.apiKeyEnv : undefined;
+    const agent = createPiRuntimeAgentSnapshot({ excludeAuthProvider: credentialName ? provider : undefined });
     let child: ChildProcessWithoutNullStreams | undefined;
     try {
+      const projectedApiKey = credentialName ? `$${credentialName}`
+        : configuredPiProviderApiKey(agent.models, provider);
       const env = buildPiChildEnv({
         provider,
-        authProviders: this.loadAuthProviders(agent.agentDir),
-        credentialValue: secretValue(this.cfg, "THT_MODEL_API_KEY"),
-        credentialFile: this.cfg.modelApiKeyFile,
-        configuredApiKey: configuredPiProviderApiKey(agent.models, provider),
+        authProviders: credentialName ? new Set() : this.loadAuthProviders(agent.agentDir),
+        credentialValue: credentialName
+          ? secretValue(this.cfg, credentialName)
+          : this.modelCatalogConfigured ? undefined : secretValue(this.cfg, "THT_MODEL_API_KEY"),
+        credentialFile: credentialName ? undefined : this.cfg.modelApiKeyFile,
+        configuredApiKey: projectedApiKey,
         additions: { THT_SESSION: sessionId, THT_AUTHOR: author },
       });
       env.PI_CODING_AGENT_DIR = agent.agentDir;
+      // A launch hint only: the gate reads the authoritative manifest before each turn.
+      delete env.THT_INTERACTION_LANGUAGE;
+      if (interactionLanguage) env.THT_INTERACTION_LANGUAGE = interactionLanguage;
       env.PI_CODING_AGENT_SESSION_DIR = agent.sessionDir;
       clearPrincipalEnvironment(env);
       if (principal) Object.assign(env, principalEnvironment(principal));
@@ -162,9 +190,12 @@ export class PiProcessManager {
     }
     const author = o.author ?? "dev@local";
     const provider = canonicalPiProvider(o.provider ?? this.cfg.defaults.provider);
+    const model = o.model ?? this.cfg.defaults.model;
     let child: ChildProcessWithoutNullStreams;
     try {
-      child = this.spawnFn(sessionId, author, provider, o.principal, o.runtimeConfig?.path);
+      child = this.spawnFn(
+        sessionId, author, provider, model, o.principal, o.runtimeConfig?.path, o.interactionLanguage,
+      );
     } catch (error) {
       o.runtimeConfig?.release();
       throw error;
@@ -235,8 +266,11 @@ export class PiProcessManager {
     const thinking = o.thinking ?? this.cfg.defaults.thinking;
 
     if (provider && model) {
+      const upstreamModel = this.modelCatalog.sessionModels()
+        .find((entry) => entry.provider === provider && entry.model === model)
+        ?.upstreamModel ?? model;
       const response = await rt.rpc.request(
-        { type: "set_model", provider, modelId: model } as object & { type: string },
+        { type: "set_model", provider, modelId: upstreamModel } as object & { type: string },
       );
       rt.bridge.setContextWindow(response?.data?.contextWindow);
     }
@@ -270,8 +304,13 @@ export class PiProcessManager {
   }
 
   async resume(sessionId: string, tht: ThtRunner): Promise<SessionRuntime> {
-    const manifest = await tht.sessionShow(sessionId) as { provider?: string; model?: string; thinking?: string } | null;
+    const manifest = await tht.sessionShow(sessionId) as {
+      provider?: string; model?: string; thinking?: string; interaction_language?: string | null;
+    } | null;
+    const language = manifest?.interaction_language
+      ?? (await tht.ensureInteractionLanguage(sessionId)).interaction_language;
     return this.spawnFor(sessionId, {
+      interactionLanguage: language,
       provider: manifest?.provider,
       model: manifest?.model,
       thinking: manifest?.thinking,

backend/src/pi/provider-smoke.ts

@@ -17,6 +17,10 @@ import {
   readConfiguredPiAgentFile,
   validateDeclarativePiConfig,
 } from "./managed-config.js";
+import {
+  loadRuntimeModelCatalog,
+  type RuntimeModelCatalog,
+} from "../models/runtime-model-catalog.js";
 
 const SMOKE_PROMPT = "Provider health check. Reply with exactly OK.";
 const SMOKE_ARGS = [
@@ -49,6 +53,7 @@ interface ProviderSmokeOptions {
   authProviders?: () => ReadonlySet<string>;
   readAuthStore?: () => string;
   readModelsStore?: () => string | undefined;
+  modelCatalog?: RuntimeModelCatalog;
 }
 
 export function createPiProviderSmoke(
@@ -65,16 +70,30 @@ export function createPiProviderSmoke(
     try {
       const canonicalProvider = canonicalPiProvider(provider);
       if (!canonicalProvider || timeoutMs <= 0) throw providerFailure();
-      const configuredAuthProviders = authProviders();
+      const configuredAuthProviders = new Set(authProviders());
       const configuredModels = options.readModelsStore
         ? options.readModelsStore()
         : readConfiguredPiAgentFile("models.json", true);
+      const catalog = options.modelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
+      const catalogConfigured = config.modelCatalogFile !== undefined
+        || catalog.defaultInteraction !== null;
+      const catalogModel = catalog.sessionModels()
+        .find((entry) => entry.provider === canonicalProvider && entry.model === model);
+      const upstreamModel = catalogModel?.upstreamModel ?? model;
+      const credentialName = catalogModel?.authentication.mode === "secret_env"
+        ? catalogModel.authentication.apiKeyEnv
+        : undefined;
+      if (credentialName) configuredAuthProviders.delete(canonicalProvider);
+      const projectedApiKey = credentialName ? `$${credentialName}`
+        : configuredPiProviderApiKey(configuredModels, canonicalProvider);
       const env = buildPiChildEnv({
         provider: canonicalProvider,
         authProviders: configuredAuthProviders,
-        credentialValue: secretValue(config, "THT_MODEL_API_KEY"),
-        credentialFile: config.modelApiKeyFile,
-        configuredApiKey: configuredPiProviderApiKey(configuredModels, canonicalProvider),
+        credentialValue: credentialName
+          ? secretValue(config, credentialName)
+          : catalogConfigured ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
+        credentialFile: credentialName ? undefined : config.modelApiKeyFile,
+        configuredApiKey: projectedApiKey,
       });
       clearPrincipalEnvironment(env);
       delete env.THT_DATA_ROOT;
@@ -113,7 +132,7 @@ export function createPiProviderSmoke(
       const capabilityGuard = failOnUnexpectedCapabilities(rpc);
       const turn = async (): Promise<void> => {
         requireSuccessfulResponse(await rpc.request({
-          type: "set_model", provider: canonicalProvider, modelId: model,
+          type: "set_model", provider: canonicalProvider, modelId: upstreamModel,
         } as object & { type: string }));
         requireSuccessfulResponse(await rpc.request({
           type: "set_thinking_level", level: reasoning,

backend/src/routes/catalog-description-consolidation.ts

@@ -9,10 +9,13 @@ import {
 } from "../catalog/types.js";
 
 const idSchema = z.uuid();
-const consolidationSchema = z.object({
-  target: z.enum(["tables", "columns"]),
-  targetIds: z.array(idSchema).min(1).max(10_000),
-}).strict();
+const consolidationSchema = z.discriminatedUnion("target", [
+  z.object({
+    target: z.enum(["tables", "columns"]),
+    targetIds: z.array(idSchema).min(1).max(10_000),
+  }).strict(),
+  z.object({ target: z.enum(["database", "database_columns"]) }).strict(),
+]);
 
 function manage(request: FastifyRequest, reply: FastifyReply) {
   return isPrincipalContext(requirePermission(request, reply, "database.manage"));
@@ -49,7 +52,7 @@ export function catalogDescriptionConsolidationRoutes(
     try {
       const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
       const input = consolidationSchema.parse(request.body);
-      const targetIds = [...new Set(input.targetIds)];
+      const targetIds = "targetIds" in input ? [...new Set(input.targetIds)] : [];
       const result = await deps.operations.run(
         databaseId,
         async () => await deps.repository.consolidateGeneratedDescriptions(

backend/src/routes/catalog-description-generation.ts

@@ -11,38 +11,35 @@ import {
   type DescriptionGenerationWorker,
 } from "../catalog/description-generation-worker.js";
 import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js";
-import { ModelCompletionProviderError } from "../catalog/model-completer.js";
 import {
-  SensitiveDataSuggestionDuplicateTargetIdsError,
-  SensitiveDataSuggestionInvalidResponseError,
-  SensitiveDataSuggestionNoEligibleColumnsError,
-  SensitiveDataSuggestionPayloadTooLargeError,
-  SensitiveDataSuggestionTargetNotFoundError,
-} from "../catalog/sensitive-data-suggester.js";
-import type { SensitiveDataSuggestionRunner } from "../catalog/sensitive-data-suggestion-runner.js";
+  SensitivityAnalysisDuplicateTargetIdsError,
+  SensitivityAnalysisInterruptedError,
+  SensitivityAnalysisNoEligibleColumnsError,
+  SensitivityAnalysisTargetNotFoundError,
+} from "../catalog/sensitivity-analysis-service.js";
+import type { SensitivityAnalysisRunner } from "../catalog/sensitivity-analysis-runner.js";
 import {
   CatalogOperationInProgressError,
+  CatalogConnectorError,
   CatalogUnavailableError,
   DescriptionGenerationRunActiveError,
   type CatalogRepository,
   type DescriptionGenerationEvent,
   type DescriptionGenerationRun,
-  type SensitiveDataSuggestionEvent,
-  type SensitiveDataSuggestionRun,
+  type SensitivityAnalysisEvent,
+  type SensitivityAnalysisRun,
 } from "../catalog/types.js";
 
 const idSchema = z.uuid();
-const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
+const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/);
 const selectedTargetIdsSchema = z.array(idSchema).min(1);
 const suggestionSchema = z.discriminatedUnion("scope", [
-  z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
+  z.object({ scope: z.literal("all") }).strict(),
   z.object({
-    modelId: modelIdSchema,
     scope: z.literal("selected_tables"),
     targetIds: selectedTargetIdsSchema,
   }).strict(),
   z.object({
-    modelId: modelIdSchema,
     scope: z.literal("selected_columns"),
     targetIds: selectedTargetIdsSchema,
   }).strict(),
@@ -112,7 +109,7 @@ function publicRun(run: DescriptionGenerationRun) {
   };
 }
 
-function publicSensitiveDataSuggestionEvent(event: SensitiveDataSuggestionEvent) {
+function publicSensitivityAnalysisEvent(event: SensitivityAnalysisEvent) {
   return {
     runId: event.runId,
     sequence: event.sequence,
@@ -122,16 +119,19 @@ function publicSensitiveDataSuggestionEvent(event: SensitiveDataSuggestionEvent)
   };
 }
 
-function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) {
+function publicSensitivityAnalysisRun(run: SensitivityAnalysisRun) {
   return {
     id: run.id,
     databaseId: run.databaseId,
     scope: run.scope,
+    engine: run.engine,
     modelId: run.modelId,
+    policyVersion: run.policyVersion,
     status: run.status,
     total: run.total,
     suggestedSensitive: run.suggestedSensitive,
     suggestedNonSensitive: run.suggestedNonSensitive,
+    unknown: run.unknown,
     inputTokens: run.inputTokens,
     cacheReadTokens: run.cacheReadTokens,
     outputTokens: run.outputTokens,
@@ -232,16 +232,10 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
   if (error instanceof CatalogUnavailableError) {
     return reply.code(503).send({
       code: "catalog_unavailable",
-      message: "The database catalog is unavailable, so no sensitive-field suggestions were prepared.",
+      message: "The database catalog is unavailable, so no sensitivity assessments were prepared.",
     });
   }
-  if (error instanceof MetadataGenerationModelUnavailableError) {
-    return reply.code(409).send({
-      code: "metadata_generation_model_unavailable",
-      message: "The selected metadata-generation model is unavailable.",
-    });
-  }
-  if (error instanceof SensitiveDataSuggestionTargetNotFoundError) {
+  if (error instanceof SensitivityAnalysisTargetNotFoundError) {
     const code = error.target === "database"
       ? "database_not_found"
       : error.target === "table"
@@ -254,45 +248,39 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
         : "One or more selected Catalog Columns were not found in this database.";
     return reply.code(404).send({ code, message });
   }
-  if (error instanceof SensitiveDataSuggestionDuplicateTargetIdsError) {
+  if (error instanceof SensitivityAnalysisDuplicateTargetIdsError) {
     return reply.code(400).send({
       code: "sensitive_data_suggestion_target_ids_duplicate",
       message: "Each selected table or column must appear only once.",
     });
   }
-  if (error instanceof SensitiveDataSuggestionNoEligibleColumnsError) {
+  if (error instanceof SensitivityAnalysisNoEligibleColumnsError) {
     return reply.code(409).send({
       code: "sensitive_data_suggestion_no_columns",
-      message: "The selected scope contains no Catalog Columns to classify.",
+      message: "The selected scope contains no Catalog Columns to assess.",
     });
   }
-  if (error instanceof SensitiveDataSuggestionPayloadTooLargeError) {
-    return reply.code(413).send({
-      code: "sensitive_data_suggestion_payload_too_large",
-      message: "The selected structural metadata cannot be divided into safe LLM requests.",
+  if (error instanceof SensitivityAnalysisInterruptedError) {
+    return reply.code(499).send({
+      code: "sensitivity_analysis_interrupted",
+      message: "Sensitivity analysis was interrupted before completion. No assessments were applied.",
     });
   }
-  if (error instanceof SensitiveDataSuggestionInvalidResponseError) {
+  if (error instanceof CatalogConnectorError) {
     return reply.code(502).send({
-      code: "sensitive_data_suggestion_invalid_response",
-      message: "The LLM returned an incomplete or invalid classification. No suggestions were applied.",
-    });
-  }
-  if (error instanceof ModelCompletionProviderError) {
-    return reply.code(502).send({
-      code: "sensitive_data_suggestion_provider_unavailable",
-      message: "The selected LLM service could not complete the request. No suggestions were applied.",
+      code: "sensitivity_source_unavailable",
+      message: "The source values could not be inspected safely. No assessments were applied.",
     });
   }
   if (error instanceof z.ZodError) {
     return reply.code(400).send({
       code: "sensitive_data_suggestion_request_invalid",
-      message: "Choose a database, one or more tables, or one or more columns to classify.",
+      message: "Choose a database, one or more tables, or one or more columns to assess.",
     });
   }
   return reply.code(500).send({
     code: "sensitive_data_suggestion_failed",
-    message: "Sensitive-field suggestions failed before review. No changes were applied.",
+    message: "Local sensitivity analysis failed before review. No changes were applied.",
   });
 }
 
@@ -300,18 +288,18 @@ function safeSuggestionHistoryError(reply: FastifyReply, error: unknown) {
   if (error instanceof CatalogUnavailableError) {
     return reply.code(503).send({
       code: "catalog_unavailable",
-      message: "Sensitive Data Suggestion history is unavailable because the database catalog is unavailable.",
+      message: "Sensitivity Analysis history is unavailable because the database catalog is unavailable.",
     });
   }
   if (error instanceof z.ZodError) {
     return reply.code(400).send({
       code: "sensitive_data_suggestion_history_request_invalid",
-      message: "Sensitive Data Suggestion history parameters are invalid.",
+      message: "Sensitivity Analysis history parameters are invalid.",
     });
   }
   return reply.code(500).send({
     code: "sensitive_data_suggestion_history_failed",
-    message: "Sensitive Data Suggestion history could not be loaded.",
+    message: "Sensitivity Analysis history could not be loaded.",
   });
 }
 
@@ -320,7 +308,7 @@ export function catalogDescriptionGenerationRoutes(
   deps: {
     repository: CatalogRepository;
     worker: DescriptionGenerationWorker;
-    sensitiveDataSuggestionRunner: SensitiveDataSuggestionRunner;
+    sensitivityAnalysisRunner: SensitivityAnalysisRunner;
   },
 ): void {
   app.post("/catalog/databases/:databaseId/sensitive-data-suggestions", async (request, reply) => {
@@ -328,16 +316,25 @@ export function catalogDescriptionGenerationRoutes(
     try {
       const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
       const input = suggestionSchema.parse(request.body);
-      const result = await deps.sensitiveDataSuggestionRunner.run(
-        databaseId,
-        input.modelId,
-        input.scope,
-        "targetIds" in input ? input.targetIds : [],
-        new AbortController().signal,
-      );
+      const controller = new AbortController();
+      const abort = () => controller.abort();
+      request.raw.once("aborted", abort);
+      reply.raw.once("close", abort);
+      let result;
+      try {
+        result = await deps.sensitivityAnalysisRunner.run(
+          databaseId,
+          input.scope,
+          "targetIds" in input ? input.targetIds : [],
+          controller.signal,
+        );
+      } finally {
+        request.raw.off("aborted", abort);
+        reply.raw.off("close", abort);
+      }
       return {
         suggestions: result.suggestions,
-        run: publicSensitiveDataSuggestionRun(result.run),
+        run: publicSensitivityAnalysisRun(result.run),
       };
     } catch (error) {
       return safeSuggestionError(reply, error);
@@ -348,8 +345,8 @@ export function catalogDescriptionGenerationRoutes(
     if (!manage(request, reply)) return reply;
     try {
       const { limit } = historyQuerySchema.parse(request.query);
-      return (await deps.repository.listSensitiveDataSuggestionRuns(limit))
-        .map(publicSensitiveDataSuggestionRun);
+      return (await deps.repository.listSensitivityAnalysisRuns(limit))
+        .map(publicSensitivityAnalysisRun);
     } catch (error) {
       return safeSuggestionHistoryError(reply, error);
     }
@@ -359,12 +356,12 @@ export function catalogDescriptionGenerationRoutes(
     if (!manage(request, reply)) return reply;
     try {
       const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
-      const run = await deps.repository.getSensitiveDataSuggestionRun(runId);
+      const run = await deps.repository.getSensitivityAnalysisRun(runId);
       if (!run) return reply.code(404).send({
         code: "sensitive_data_suggestion_run_not_found",
-        message: "Sensitive Data Suggestion Run was not found.",
+        message: "Sensitivity Analysis Run was not found.",
       });
-      return publicSensitiveDataSuggestionRun(run);
+      return publicSensitivityAnalysisRun(run);
     } catch (error) {
       return safeSuggestionHistoryError(reply, error);
     }
@@ -375,14 +372,14 @@ export function catalogDescriptionGenerationRoutes(
     try {
       const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
       const { after } = eventQuerySchema.parse(request.query);
-      if (!(await deps.repository.getSensitiveDataSuggestionRun(runId))) {
+      if (!(await deps.repository.getSensitivityAnalysisRun(runId))) {
         return reply.code(404).send({
           code: "sensitive_data_suggestion_run_not_found",
-          message: "Sensitive Data Suggestion Run was not found.",
+          message: "Sensitivity Analysis Run was not found.",
         });
       }
-      return (await deps.repository.listSensitiveDataSuggestionEvents(runId, after))
-        .map(publicSensitiveDataSuggestionEvent);
+      return (await deps.repository.listSensitivityAnalysisEvents(runId, after))
+        .map(publicSensitivityAnalysisEvent);
     } catch (error) {
       return safeSuggestionHistoryError(reply, error);
     }

backend/src/routes/catalog-schema.ts

@@ -19,8 +19,14 @@ const metadataSchema = z.object({
   description: z.string().max(20_000).nullable().optional(),
   generatedDescription: z.string().max(20_000).nullable().optional(),
   sensitive: z.boolean().optional(),
+  sensitivityReason: z.string().max(2_000).nullable().optional(),
 }).strict().refine((value) => (
-  "description" in value || "generatedDescription" in value || "sensitive" in value
+  "description" in value
+  || "generatedDescription" in value
+  || "sensitive" in value
+  || "sensitivityReason" in value
+)).refine((value) => (
+  value.sensitivityReason == null || value.sensitive === true
 ));
 const createRunSchema = z.object({
   version: z.number().int().positive(),
@@ -56,7 +62,10 @@ function safeError(reply: FastifyReply, error: unknown) {
     return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
   }
   if (error instanceof CatalogConnectorError) {
-    return reply.code(502).send({ code: "schema_introspection_failed", message: "The database schema could not be read safely." });
+    return reply.code(502).send({
+      code: "schema_introspection_failed",
+      message: "The database schema could not be read. Check the connection and credentials, then try again.",
+    });
   }
   if (error instanceof z.ZodError) {
     return reply.code(400).send({ code: "schema_request_invalid", message: "Schema request is invalid." });
@@ -113,6 +122,12 @@ export function catalogSchemaRoutes(
         if (current.version !== input.version) {
           return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
         }
+        const nextSensitive = input.sensitive ?? current.sensitive;
+        const nextSensitivityReason = nextSensitive
+          ? ("sensitivityReason" in input
+              ? normalized(input.sensitivityReason ?? null)
+              : current.sensitivityReason)
+          : null;
         const updated = await deps.repository.updateColumnMetadata(
           databaseId,
           tableId,
@@ -123,6 +138,7 @@ export function catalogSchemaRoutes(
             ? normalized(input.generatedDescription ?? null)
             : current.generatedDescription,
           input.sensitive,
+          nextSensitivityReason,
         );
         if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
         return updated;

backend/src/routes/evidence.ts

@@ -0,0 +1,90 @@
+import path from "node:path";
+import type { FastifyInstance } from "fastify";
+import { z } from "zod";
+import { requirePermission, isPrincipalContext } from "../auth/authorization.js";
+import type { ThtRunner } from "../tht/tht-runner.js";
+import type { WorkspaceRegistry } from "../workspaces/registry.js";
+import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
+
+const params = z.object({ workspaceId: z.string().regex(/^[a-z][a-z0-9-]{2,62}$/),
+  evidenceId: z.string().regex(/^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$/).optional() });
+const query = z.object({
+  q: z.string().max(1000).optional(), kind: z.enum(["domain", "glossary", "enum", "example", "mapping", "normalization", "formula", "reference"]).optional(),
+  purpose: z.enum(["disambiguation", "rewriting", "schema_linking", "sql_generation"]).optional(),
+  status: z.enum(["new", "modified", "active", "removed", "review_required", "legacy", "invalid"]).optional(),
+  concept: z.string().max(300).optional(), table: z.string().max(300).optional(), column: z.string().max(300).optional(),
+  source: z.string().max(300).optional(), language: z.string().max(30).optional(),
+  sort: z.enum(["title", "id", "kind", "status"]).optional(), direction: z.enum(["asc", "desc"]).optional(),
+  page: z.coerce.number().int().positive().optional(), page_size: z.coerce.number().int().min(1).max(100).optional(),
+}).strict();
+const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
+
+export function evidenceRoutes(app: FastifyInstance, deps: {
+  runner: Pick<ThtRunner, "withPrincipal">; registry: Pick<WorkspaceRegistry, "list">;
+  registryRoot: string; hostRegistryRoot?: string;
+  service: Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
+}) {
+  app.post("/workspaces/:workspaceId/evidence/sources", async (request, reply) => {
+    const principal = requirePermission(request, reply, "evidence.manage");
+    if (!isPrincipalContext(principal)) return reply;
+    try {
+      const { workspaceId } = params.parse(request.params);
+      const body = z.discriminatedUnion("action", [
+        z.object({ action: z.literal("refresh") }).strict(),
+        z.object({ action: z.literal("decide"), sourceId: z.string().regex(/^[a-f0-9]{64}$/),
+          revision: z.string().regex(/^[a-f0-9]{64}$/), decision: z.enum(["keep", "replace"]) }).strict(),
+      ]).parse(request.body);
+      if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
+      if (!deps.service.evidenceSources) throw new Error("Evidence service unavailable");
+      return await deps.service.evidenceSources({ workspaceId, ...body, actor: principal.subject });
+    } catch (error) {
+      return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
+        message: error instanceof z.ZodError ? "Invalid source request." : "Evidence source service is unavailable." });
+    }
+  });
+  app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence", read);
+  app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence/:evidenceId", read);
+  async function read(request: import("fastify").FastifyRequest, reply: import("fastify").FastifyReply) {
+    const principal = requirePermission(request, reply, "evidence.manage");
+    if (!isPrincipalContext(principal)) return reply;
+    try {
+      const { workspaceId, evidenceId } = params.parse(request.params);
+      const filters = query.parse(request.query);
+      if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
+      const root = path.join(deps.registryRoot, "repo", workspaceId);
+      const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
+        ["evidence", "admin", "--workspace", workspaceId],
+        JSON.stringify({ root, query: { ...filters, ...(evidenceId ? { id: evidenceId } : {}) } }),
+      );
+      const payload = JSON.parse(result.stdout);
+      if (result.code !== 0) return reply.code(503).send(payload);
+      if (evidenceId && !payload.item) return reply.code(404).send({ code: "evidence_not_found", message: "Evidence was not found." });
+      const host = deps.hostRegistryRoot;
+      const hostPath = host && (path.isAbsolute(host) || path.win32.isAbsolute(host))
+        ? (path.win32.isAbsolute(host) && !path.isAbsolute(host) ? path.win32 : path).join(host, "repo") : null;
+      const hostJoin = hostPath && path.win32.isAbsolute(hostPath) && !path.isAbsolute(hostPath) ? path.win32.join : path.join;
+      return { ...payload, location: { repository: hostPath, workspace: hostPath ? hostJoin(hostPath, workspaceId) : null,
+        runtime_workspace: root, host: "Installation host", command: `tht workspace evidence consolidate --workspace ${workspaceId}`,
+        git_commands: hostPath ? [
+          `git -C ${quote(hostPath)} status --short -- ${quote(workspaceId + "/evidence")}`,
+          `git -C ${quote(hostPath)} diff -- ${quote(workspaceId + "/evidence")}`,
+          `git -C ${quote(hostPath)} add -A -- ${quote(workspaceId + "/evidence")}`,
+          `git -C ${quote(hostPath)} commit --only -m 'Curate Evidence' -- ${quote(workspaceId + "/evidence")}`,
+          `git -C ${quote(hostPath)} push`,
+        ] : [] } };
+    } catch (error) {
+      return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
+        message: error instanceof z.ZodError ? "Evidence filters are invalid." : "Evidence archive is unavailable." });
+    }
+  }
+  app.post("/workspaces/:workspaceId/evidence/consolidate", async (request, reply) => {
+    const principal = requirePermission(request, reply, "evidence.manage");
+    if (!isPrincipalContext(principal)) return reply;
+    try {
+      const { workspaceId } = params.parse(request.params);
+      if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
+      if (!deps.service.consolidateEvidence) throw new Error("Evidence service unavailable");
+      return await deps.service.consolidateEvidence({ workspaceId });
+    } catch { return reply.code(503).send({ code: "evidence_unavailable", message: "Evidence consolidation is unavailable." }); }
+  });
+}

backend/src/routes/memory.ts

@@ -0,0 +1,94 @@
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
+import type { ThtRunner } from "../tht/tht-runner.js";
+import type { WorkspaceRegistry } from "../workspaces/registry.js";
+import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
+
+const paramsSchema = z.object({
+  workspaceId: z.string().regex(/^[a-z][a-z0-9_-]{0,63}$/),
+  cardId: z.string().regex(/^mem-[0-9a-f-]{36}$/).optional(),
+});
+const family = z.enum(["domain_clarification", "sql_rule", "solved_question", "explained_error"]);
+const cardSchema = z.object({
+  family, subject: z.string().trim().min(1).max(1000),
+  detail: z.string().max(50000).default(""), scope: z.string().trim().min(1).max(10000),
+  rationale: z.string().max(10000).default(""), question: z.string().max(10000).default(""),
+  sql: z.string().max(100000).default(""),
+  concepts: z.array(z.string().trim().min(1).max(200)).max(100).default([]),
+  dependencies: z.array(z.object({
+    database: z.string().trim().min(1).max(200), schema_name: z.string().max(200).default(""),
+    table: z.string().max(200).default(""), column: z.string().max(200).default(""),
+  }).strict()).max(200).default([]),
+  links: z.array(z.object({
+    target_id: z.string().min(1).max(100), meaning: z.string().trim().min(1).max(1000),
+  }).strict()).max(200).default([]),
+}).strict();
+const querySchema = z.object({
+  q: z.string().max(1000).optional(), family: family.optional(),
+  concept: z.string().max(200).optional(), database: z.string().max(200).optional(),
+  table: z.string().max(200).optional(), column: z.string().max(200).optional(),
+  origin: z.enum(["manual", "workflow"]).optional(),
+  updated_after: z.iso.datetime({ offset: true }).optional(),
+  updated_before: z.iso.datetime({ offset: true }).optional(),
+  page: z.coerce.number().int().positive().optional(),
+  page_size: z.coerce.number().int().min(1).max(100).optional(),
+  sort: z.enum(["updated_at", "created_at", "subject", "family"]).optional(),
+  direction: z.enum(["asc", "desc"]).optional(),
+}).strict();
+
+export function memoryRoutes(app: FastifyInstance, deps: {
+  runner: Pick<ThtRunner, "withPrincipal">;
+  registry: Pick<WorkspaceRegistry, "list" | "read">;
+  runtime: SemanticRuntimeConfig;
+}) {
+  const invoke = (action: string) => async (request: FastifyRequest, reply: FastifyReply) => {
+    const principal = requirePermission(request, reply, "memory.manage");
+    if (!isPrincipalContext(principal)) return reply;
+    try {
+      const { workspaceId, cardId } = paramsSchema.parse(request.params);
+      const input = action === "list" ? querySchema.parse(request.query)
+        : action === "create" || action === "update"
+          ? { card: cardSchema.parse(request.body), ...(cardId ? { id: cardId } : {}) }
+          : cardId ? { id: cardId } : {};
+      // Registry identity is enough: administrative browsing must not require a DWH binding.
+      const workspaces = await deps.registry.list();
+      if (!workspaces.some(workspace => workspace.id === workspaceId)) {
+        return reply.code(404).send({ code: "workspace_invalid", message: "Workspace was not found." });
+      }
+      const { workspace } = await deps.registry.read(workspaceId);
+      const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
+        ["memory", "admin", "--workspace", workspaceId],
+        JSON.stringify({ action, request: input, runtime: {
+          ...deps.runtime, memoryLanguage: workspace.workspace.language,
+        } }),
+      );
+      let payload;
+      try { payload = JSON.parse(result.stdout); } catch {
+        return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
+      }
+      if (result.code !== 0) {
+        const codes: Record<string, number> = {
+          memory_invalid: 400, memory_forbidden: 403, memory_not_found: 404,
+          memory_conflict: 409, memory_unavailable: 503,
+        };
+        const code = typeof payload?.code === "string" && payload.code in codes
+          ? payload.code : "memory_unavailable";
+        return reply.code(codes[code]).send({ code, message: "Memory operation could not be completed." });
+      }
+      return reply.code(action === "create" ? 201 : 200).send(payload);
+    } catch (error) {
+      if (error instanceof z.ZodError) {
+        return reply.code(400).send({ code: "memory_invalid", message: "Memory request is invalid." });
+      }
+      return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
+    }
+  };
+  app.get("/workspaces/:workspaceId/memory", invoke("list"));
+  app.post("/workspaces/:workspaceId/memory", invoke("create"));
+  app.get("/workspaces/:workspaceId/memory/pending", invoke("pending"));
+  app.get("/workspaces/:workspaceId/memory/:cardId", invoke("show"));
+  app.put("/workspaces/:workspaceId/memory/:cardId", invoke("update"));
+  app.delete("/workspaces/:workspaceId/memory/:cardId", invoke("delete"));
+  app.post("/workspaces/:workspaceId/memory/:cardId/retry", invoke("retry"));
+}

backend/src/routes/meta.ts

@@ -1,10 +1,8 @@
 import { readdirSync } from "node:fs";
 import { join } from "node:path";
 import type { FastifyInstance } from "fastify";
-import type { PiModel } from "../pi/list-models.js";
 import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
-
-export type ListModelsFn = () => Promise<PiModel[]>;
+import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
 
 /**
  * List YAML workspace configs found in <harnessDir>/workspaces/*.yaml.
@@ -25,20 +23,17 @@ export function listWorkspaces(harnessDir: string): { name: string; file: string
 
 export function metaRoutes(
   app: FastifyInstance,
-  deps: { harnessDir: string; listModels?: ListModelsFn },
+  deps: { harnessDir: string; modelCatalog: RuntimeModelCatalog },
 ): void {
   app.get("/models", async (request, reply) => {
     if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply;
-    const fn = deps.listModels ?? (async () => []);
-    try {
-      return { models: await fn() };
-    } catch (error) {
-      app.log.warn({
-        component: "pi-model-list",
-        errorType: error instanceof Error ? error.name : typeof error,
-      }, "Pi model listing failed");
-      // Graceful fallback: Pi may not be running; don't crash the server.
-      return { models: [] as PiModel[] };
-    }
+    return {
+      models: deps.modelCatalog.sessionModels().map((entry) => ({
+        provider: entry.provider,
+        id: entry.model,
+        name: entry.label,
+        reasoning: entry.session?.reasoning ?? false,
+      })),
+    };
   });
 }

backend/src/routes/pi-management.ts

@@ -11,13 +11,6 @@ export function piManagementRoutes(
   deps: { service: PiManagementService },
 ): void {
   app.get("/pi-management/status", async (request, reply) => run(request, reply, deps, () => deps.service.status()));
-  app.get("/pi-management/options", async (request, reply) => run(request, reply, deps, () => deps.service.options()));
-  app.put("/pi-management/config", async (request, reply) => run(
-    request,
-    reply,
-    deps,
-    () => deps.service.configure((request.body ?? {}) as Record<string, unknown>),
-  ));
   app.post("/pi-management/test", async (request, reply) => run(request, reply, deps, () => deps.service.test()));
   app.get("/pi-management/logs", async (request, reply) => run(request, reply, deps, () => deps.service.logs()));
 }
@@ -38,8 +31,7 @@ async function run<T>(
     return await action();
   } catch (error) {
     if (error instanceof PiManagementError) {
-      const statusCode = error.code === "pi_management_invalid_config" ? 400 : 503;
-      return reply.code(statusCode).send({ code: error.code, error: error.message });
+      return reply.code(503).send({ code: error.code, error: error.message });
     }
     return reply.code(503).send({ code: "pi_management_unavailable", error: "Pi management is unavailable" });
   }

backend/src/routes/sessions.ts

@@ -6,12 +6,14 @@ import type { Settings } from "../settings/settings-store.js";
 import { getPrincipal } from "../auth/auth.js";
 import type { PrincipalContext } from "../auth/principal.js";
 import type { ReadinessManager } from "../runtime/readiness-manager.js";
-import type { ListModelsFn } from "./meta.js";
+import type { ListModelsFn } from "../pi/list-models.js";
 import type { WorkspaceRegistry } from "../workspaces/registry.js";
 import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js";
 import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
 import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
-import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js";
+import { splitCanonicalModelId, type RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
+import type { CatalogRepository } from "../catalog/types.js";
+import { interactionLanguage } from "../tht/interaction-language.js";
 
 const BOOTSTRAP_FAILURE_MESSAGE =
   "Session startup failed. Check configuration and connectivity, then Resume the session.";
@@ -41,11 +43,40 @@ export function sessionRoutes(
     /** Fail-closed installation/runtime transport capability check. */
     workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean;
     maintenanceBarrier: MaintenanceBarrier;
-    /** Optional only for narrow route-test stubs and installations without a Catalog database. */
-    effectiveRelationships?: EffectiveRelationshipSnapshotProvider;
+    modelCatalog: RuntimeModelCatalog;
+    /** PostgreSQL authority for mandatory preprocessing admission. */
+    catalogRepository?: CatalogRepository;
   },
 ) {
   const lifecycleTails = new Map<string, Promise<void>>();
+
+  const preprocessingIsCurrent = async (
+    workspaceId: string,
+    inputFingerprint?: string,
+  ): Promise<boolean> => {
+    if (!d.catalogRepository) return true;
+    const database = await d.catalogRepository.getByWorkspace(workspaceId);
+    return database !== undefined
+      && database.preprocessingStatus === "succeeded"
+      && database.preprocessedMetadataRevision === database.metadataContentRevision
+      && (inputFingerprint === undefined
+        || database.preprocessingInputFingerprint === inputFingerprint);
+  };
+
+  const catalogTransportSupportsSessionRuntime = async (workspaceId: string): Promise<boolean> => {
+    if (!d.catalogRepository) return true;
+    const database = await d.catalogRepository.getByWorkspace(workspaceId);
+    return database === undefined || database.binding.transport !== "ssh_tunnel";
+  };
+
+  const currentInputFingerprint = async (
+    runner: any,
+    workspaceConfigPath: string,
+  ): Promise<string | undefined> => (
+    typeof runner.workspaceInputFingerprint === "function"
+      ? await runner.workspaceInputFingerprint(workspaceConfigPath)
+      : undefined
+  );
   const boundRuntimes = new Map<
     string,
     ReturnType<PiProcessManager["createFor"]>
@@ -90,16 +121,13 @@ export function sessionRoutes(
   const optionsWithRuntimeConfig = async (
     runner: any,
     workspaceConfigPath: string | undefined,
-    workspaceId: string | undefined,
+    _workspaceId: string | undefined,
     options: any,
   ) => {
     if (!workspaceConfigPath || typeof runner.acquireWorkspaceRuntime !== "function") return options;
-    const effectiveRelationships = workspaceId && d.effectiveRelationships
-      ? await d.effectiveRelationships.render(workspaceId)
-      : undefined;
     return {
       ...options,
-      runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath, effectiveRelationships),
+      runtimeConfig: await runner.acquireWorkspaceRuntime(workspaceConfigPath),
     };
   };
 
@@ -123,13 +151,34 @@ export function sessionRoutes(
   });
   app.addHook("onResponse", async (req) => { admissionLeases.get(req)?.(); });
 
+  type SessionRevisionScan = {
+    revisions: Awaited<ReturnType<typeof d.workspaceRegistry.list>>;
+    retainedComplete: boolean;
+  };
+  let retainedSnapshotWarning: string | null = null;
+
   /** Include retained historical descriptors so removed workspaces remain resumable. */
-  const sessionRevisions = async () => {
+  const sessionRevisions = async (): Promise<SessionRevisionScan> => {
     const registry = d.workspaceRegistry as Partial<WorkspaceRegistry>;
     if (typeof registry.listRetainedSnapshots === "function") {
-      return await registry.listRetainedSnapshots();
+      try {
+        const revisions = await registry.listRetainedSnapshots();
+        retainedSnapshotWarning = null;
+        return { revisions, retainedComplete: true };
+      } catch (error) {
+        // A legacy/corrupt historical snapshot must not make active sessions (and their SSE
+        // reviewer gates) unreachable. The registry still rejects that snapshot; this fallback
+        // exposes only descriptors from the verified active state and deliberately disables
+        // retention reconciliation because the resulting session view is incomplete.
+        const detail = error instanceof Error ? error.message : "unknown error";
+        if (retainedSnapshotWarning !== detail) {
+          console.warn("[sessions] retained snapshot discovery failed; using active snapshots:", detail);
+          retainedSnapshotWarning = detail;
+        }
+        return { revisions: await d.workspaceRegistry.list(), retainedComplete: false };
+      }
     }
-    return await d.workspaceRegistry.list();
+    return { revisions: await d.workspaceRegistry.list(), retainedComplete: true };
   };
 
   const isNotFound = (error: unknown) =>
@@ -173,7 +222,7 @@ export function sessionRoutes(
     };
     let revisions: Awaited<ReturnType<typeof d.workspaceRegistry.list>>;
     try {
-      revisions = await sessionRevisions();
+      ({ revisions } = await sessionRevisions());
     } catch (registryError) {
       // Sessions created before revision pinning still live under the installation's legacy
       // default config. Keep that compatibility path available when a fresh installation has
@@ -333,8 +382,17 @@ export function sessionRoutes(
   app.post("/sessions", async (req, reply) => {
     const b = req.body as {
       question: string; name?: string; workspace?: string; workspaceId?: string;
-      provider?: string; model?: string; thinking?: string;
+      provider?: string; model?: string; thinking?: string; interactionLanguage?: unknown;
     };
+    const language = interactionLanguage(b?.interactionLanguage);
+    if (!language) return reply.code(400).send({
+      code: "invalid_interaction_language",
+      error: "interactionLanguage must be a well-formed BCP-47 language tag",
+    });
+    if ((b.provider === undefined) !== (b.model === undefined)
+      || (b.provider !== undefined && (typeof b.provider !== "string" || typeof b.model !== "string" || !b.provider || !b.model))) {
+      return reply.code(400).send({ error: "provider and model must be supplied together" });
+    }
     const principal = getPrincipal(req);
     let s: Settings;
     try { s = await d.getSettings(principal); } catch { return storageFailure(reply); }
@@ -358,7 +416,6 @@ export function sessionRoutes(
       let workspaceId: string | undefined;
       let workspaceRevision: string | undefined;
       let workspaceDescriptor: WorkspaceDescriptor | undefined;
-      let allowedModels: readonly string[] | undefined;
       if (requestedWorkspaceId) {
         try {
           const registry = d.workspaceRegistry as Partial<WorkspaceRegistry>;
@@ -378,7 +435,19 @@ export function sessionRoutes(
           workspaceId = resolved.revision.id;
           workspaceRevision = resolved.revision.commit;
           workspaceDescriptor = resolved.workspace;
-          allowedModels = resolved.workspace.llm_policy.allowed;
+          if (!await catalogTransportSupportsSessionRuntime(workspaceId)) {
+            return reply.code(409).send({
+              error: "This workspace transport is not available to runtime sessions.",
+              code: "workspace_not_activatable",
+            });
+          }
+          const fingerprint = await currentInputFingerprint(runner, workspaceConfigPath);
+          if (!await preprocessingIsCurrent(workspaceId, fingerprint)) {
+            return reply.code(409).send({
+              error: "Run workspace preprocessing before starting the core.",
+              code: "preprocessing_required",
+            });
+          }
         } catch {
           return reply.code(409).send({
             error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE,
@@ -386,12 +455,15 @@ export function sessionRoutes(
           });
         }
       }
-      const provider = b.provider ?? s.provider;
-      const model = b.model ?? s.model;
+	  const requestedCanonical = b.provider && b.model ? `${b.provider}/${b.model}` : undefined;
+	  const selectedCanonical = requestedCanonical ?? d.modelCatalog.defaultInteraction;
+	  if (selectedCanonical && d.modelCatalog.defaultInteraction && !d.modelCatalog.hasSession(selectedCanonical)) {
+	    return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
+	  }
+	  const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : undefined;
+	  const provider = selected?.provider ?? b.provider;
+	  const model = selected?.model ?? b.model;
       const thinking = b.thinking ?? s.thinking;
-      if (allowedModels && provider && model && !allowedModels.includes(`${provider}/${model}`)) {
-        return reply.code(400).send({ error: "Selected model is not allowed by this workspace." });
-      }
       // A persisted session is resumable without keeping Pi alive. New work replaces every
       // runtime owned by this principal, while runtimes belonging to other users remain intact.
       // Optional chaining preserves the deliberately narrow manager stubs used by route tests.
@@ -437,11 +509,15 @@ export function sessionRoutes(
       // registry snapshot. The legacy fallback stays available for sessions created before
       // the browser-local preference migration.
       let id: string;
+      let sessionLanguage = language;
       try {
-        ({ id } = await runner.sessionNew({
+        const created = await runner.sessionNew({
           question: b.question, name: b.name, workspaceConfigPath,
           workspaceId, workspaceRevision, provider, model, thinking,
-        }));
+          interactionLanguage: language,
+        });
+        id = created.id;
+        sessionLanguage = created.interaction_language ?? language;
         manifestPersisted = true;
         if (revisionLease) {
           await revisionLease.markPersisted().catch((error: unknown) => {
@@ -454,6 +530,7 @@ export function sessionRoutes(
       } catch { return storageFailure(reply); }
       const options = {
         provider, model, thinking,
+        interactionLanguage: sessionLanguage,
         author: principal.displayName ?? principal.subject,
         principal,
         question: b.question,
@@ -516,8 +593,8 @@ export function sessionRoutes(
         ? { ...principal, isAdmin: false }
         : ownershipPrincipal(principal, "session.read_all");
       const runner = runnerFor(scopedPrincipal);
-      const revisions = await sessionRevisions();
-      const lists = await Promise.all(revisions
+      const revisionScan = await sessionRevisions();
+      const lists = await Promise.all(revisionScan.revisions
         .map((revision) => runner.sessionList(revision.snapshotPath) as Promise<SessionRow[]>));
       const sessions = new Map<string, SessionRow>();
       for (const row of lists.flat()) {
@@ -527,7 +604,8 @@ export function sessionRoutes(
       // Only an administrator-visible complete list (or the single local principal) is safe
       // input for retention. A remote per-user view can never discard another principal's pin.
       const reconcileSnapshotRetention = (d.workspaceRegistry as Partial<WorkspaceRegistry>).reconcileSnapshotRetention;
-      const hasCompleteRetentionView = (scope === "all" || principal.issuer === "local")
+      const hasCompleteRetentionView = revisionScan.retainedComplete
+        && (scope === "all" || principal.issuer === "local")
         && hasPermission(principal, "session.read_all");
       if (hasCompleteRetentionView && typeof reconcileSnapshotRetention === "function") {
         const retained = [...new Set(list
@@ -561,6 +639,23 @@ export function sessionRoutes(
     } catch (error) { return lifecycleFailure(reply, error); }
     const rt = d.mgr.get(id);
     if (!rt) return reply.code(404).send({ error: "sessione non attiva" });
+    const pending = rt.bridge.pendingWidget?.() as any;
+    const response = (req.body as any)?.ui_response;
+    if (pending?.widget === "archive-repair" && response?.id === pending.id && !response.control) {
+      const choices = response.choices;
+      if (!Array.isArray(choices) || choices.length !== 1)
+        return reply.code(400).send({ error: "Select one archive repair choice" });
+      if (choices[0] !== "continue" && choices[0] !== "reject") {
+        const option = pending.repair?.options?.find((item: any) => item.id === choices[0]);
+        if (!option || !["memory", "evidence"].includes(option.archive))
+          return reply.code(400).send({ error: "Unknown archive repair choice" });
+        const permission = option.archive === "memory" ? "memory.manage" : "evidence.manage";
+        if (!principal.isAdmin || !hasPermission(principal, permission))
+          return reply.code(403).send({ error: "Archive corrections require an administrator" });
+        if (rt.ownerKey !== `${principal.issuer}\0${principal.subject}`)
+          return reply.code(409).send({ error: "Resume the session with your account before correcting an archive" });
+      }
+    }
     if (!rt.bridge.respond((req.body as any).ui_response)) {
       return reply.code(409).send({ error: "risposta non corrispondente al gate in attesa" });
     }
@@ -580,6 +675,13 @@ export function sessionRoutes(
   app.post("/sessions/:id/resume", async (req, reply) => {
     const id = (req.params as any).id;
     const principal = getPrincipal(req);
+    if (Object.hasOwn(req.body ?? {}, "interactionLanguage")
+      || Object.hasOwn(req.body ?? {}, "interaction_language")) {
+      return reply.code(400).send({
+        code: "interaction_language_pinned",
+        error: "Resume uses the session's persisted interaction language; overrides are not accepted",
+      });
+    }
     return withSessionLifecycle(id, async () => {
       let settings: Settings;
       let located: LocatedSession | undefined;
@@ -597,7 +699,19 @@ export function sessionRoutes(
       const saved = manifest as {
         provider?: string; model?: string; thinking?: string;
         workspace_id?: string; workspace_revision?: string;
+        interaction_language?: string | null;
       };
+	  const requested = (req.body ?? {}) as { provider?: string; model?: string; thinking?: string };
+	  if ((requested.provider === undefined) !== (requested.model === undefined)
+	    || (requested.provider !== undefined && (typeof requested.provider !== "string" || typeof requested.model !== "string" || !requested.provider || !requested.model))) {
+	    return reply.code(400).send({ error: "provider and model must be supplied together" });
+	  }
+	  const selectedCanonical = requested.provider && requested.model
+	    ? `${requested.provider}/${requested.model}` : d.modelCatalog.defaultInteraction;
+	  if (d.modelCatalog.defaultInteraction && (!selectedCanonical || !d.modelCatalog.hasSession(selectedCanonical))) {
+	    return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
+	  }
+	  const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : saved;
       let workspaceConfigPath: string;
       let workspaceDescriptor: WorkspaceDescriptor | undefined;
       try {
@@ -606,14 +720,37 @@ export function sessionRoutes(
         workspaceDescriptor = resolved.workspace;
       }
       catch { return unavailableWorkspaceReply(reply); }
+      if (d.catalogRepository && saved.workspace_id
+        && !await catalogTransportSupportsSessionRuntime(saved.workspace_id)) {
+        return reply.code(409).send({
+          error: "This workspace transport is not available to runtime sessions.",
+          code: "workspace_not_activatable",
+        });
+      }
+      const fingerprint = d.catalogRepository
+        ? await currentInputFingerprint(runner, workspaceConfigPath)
+        : undefined;
+      if (d.catalogRepository
+        && (!saved.workspace_id || !await preprocessingIsCurrent(saved.workspace_id, fingerprint))) {
+        return reply.code(409).send({
+          error: "Run workspace preprocessing before starting the core.",
+          code: "preprocessing_required",
+        });
+      }
       try { settings = await d.getSettings(principal); } catch { return storageFailure(reply); }
+      let language = saved.interaction_language;
+      if (language == null) {
+        try {
+          language = (await runner.ensureInteractionLanguage(id, workspaceConfigPath)).interaction_language;
+        } catch { return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE }); }
+      }
       // This check belongs inside the per-session lock: a preceding cold Resume may have
       // installed a running runtime while this request was waiting.
       const existing = d.mgr.get(id);
       if (existing) {
         const state = existing.bridge.turnState();
         if (state === "running" || state === "waiting") {
-          return reply.code(200).send({ id, alreadyActive: true });
+          return reply.code(200).send({ id, alreadyActive: true, workspaceId: saved.workspace_id });
         }
       }
       const ensure = await d.readiness.ensure(
@@ -624,8 +761,9 @@ export function sessionRoutes(
         ...(ensure.code ? { code: ensure.code } : {}),
       });
       const options = {
-        provider: saved?.provider,
-        model: saved?.model,
+        provider: selected.provider,
+        interactionLanguage: language,
+        model: selected.model,
         thinking: saved?.thinking ?? settings.thinking,
         author: principal.displayName ?? principal.subject,
         principal,
@@ -647,7 +785,7 @@ export function sessionRoutes(
       if (current) {
         const state = current.bridge.turnState();
         if (state === "running" || state === "waiting") {
-          return reply.code(200).send({ id, alreadyActive: true });
+          return reply.code(200).send({ id, alreadyActive: true, workspaceId: saved.workspace_id });
         }
       }
 
@@ -688,7 +826,7 @@ export function sessionRoutes(
         d.mgr.configure(rt, runtimeOptions), null,
         () => d.mgr.start(id, rt, runtimeOptions),
       );
-      return reply.code(200).send({ id, alreadyActive: false });
+      return reply.code(200).send({ id, alreadyActive: false, workspaceId: saved.workspace_id });
     });
   });
   app.post("/sessions/:id/close", async (req, reply) => {

backend/src/routes/settings.ts

@@ -1,17 +1,27 @@
 import type { FastifyInstance } from "fastify";
 import type { AppConfig } from "../config.js";
 import type { Settings } from "../settings/settings-store.js";
-import { listWorkspaces, type ListModelsFn } from "./meta.js";
+import { listWorkspaces } from "./meta.js";
 import type { PrincipalContext } from "../auth/principal.js";
 import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
+import {
+  splitCanonicalModelId,
+  type RuntimeModelCatalog,
+} from "../models/runtime-model-catalog.js";
 
-/** Merge stored settings over env/first-workspace defaults. */
-export function effectiveSettings(cfg: AppConfig, stored: Settings): Settings {
+/** Merge only workspace and runtime-thinking preferences; model defaults belong to modelCatalog. */
+export function effectiveSettings(
+  cfg: AppConfig,
+  stored: Settings,
+  modelCatalog?: RuntimeModelCatalog,
+): Settings {
   const workspaces = listWorkspaces(cfg.harnessDir);
+  const selected = modelCatalog?.defaultInteraction
+    ? splitCanonicalModelId(modelCatalog.defaultInteraction)
+    : undefined;
   return {
     workspace: stored.workspace ?? workspaces[0]?.name,
-    provider: cfg.defaults.provider ?? stored.provider,
-    model: cfg.defaults.model ?? stored.model,
+    ...(selected ?? {}),
     thinking: cfg.defaults.thinking ?? stored.thinking,
   };
 }
@@ -19,7 +29,7 @@ export function effectiveSettings(cfg: AppConfig, stored: Settings): Settings {
 export function settingsRoutes(
   app: FastifyInstance,
   deps: {
-    cfg: AppConfig; listModels: ListModelsFn;
+    cfg: AppConfig;
     getSettings: (principal: PrincipalContext) => Promise<Settings>;
   },
 ): void {
@@ -37,22 +47,6 @@ export function settingsRoutes(
     const principal = requirePermission(req, reply, "settings.manage");
     if (!isPrincipalContext(principal)) return principal;
     const b = (req.body ?? {}) as Settings;
-    if (b.model) {
-      let available: { provider: string; id: string }[] = [];
-      try {
-        available = await deps.listModels();
-      } catch {
-        available = [];
-      }
-      // Only validate when Pi gave us a non-empty list; otherwise allow (degraded).
-      if (available.length > 0 && !available.some(
-        (candidate) => candidate.provider === b.provider && candidate.id === b.model,
-      )) {
-        return reply.code(400).send({
-          error: `Unknown model: ${b.provider ?? "unknown"}/${b.model}`,
-        });
-      }
-    }
     try {
       // Retain this endpoint as a validating compatibility surface for older clients, but do
       // not write anonymous users' choices to shared server storage.

backend/src/routes/workspace-preprocessing.ts

@@ -0,0 +1,402 @@
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
+import {
+  CatalogUnavailableError,
+  type CatalogRepository,
+  type WorkspaceDatabase,
+} from "../catalog/types.js";
+import type { ThtRunner } from "../tht/tht-runner.js";
+import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
+import type { PreprocessingJobState } from "../workspaces/preprocessing-state.js";
+import type { WorkspaceRegistry } from "../workspaces/registry.js";
+
+const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
+const ACTIVE_SYNC_STATES = new Set(["queued", "running", "awaiting_confirmation", "applying"]);
+
+export type WorkspacePreprocessingUiState =
+  | "ready"
+  | "required"
+  | "running"
+  | "blocked"
+  | "failed";
+
+export interface WorkspacePreprocessingStatus {
+  schemaVersion: 1;
+  workspaceId: string;
+  state: WorkspacePreprocessingUiState;
+  actionable: boolean;
+  clearable: boolean;
+  detail: string;
+  reason?: string;
+  nextStep?: string;
+  metadataRevision?: number;
+  preprocessedMetadataRevision?: number;
+  startedAt?: string;
+  finishedAt?: string;
+  progress?: {
+    stage: "catalog_snapshot" | "schema_index" | "evidence" | "finalizing";
+    step: number;
+    totalSteps: 4;
+  };
+  lastFailure?: {
+    stage: string;
+    errorCode: string;
+    finishedAt: string;
+  };
+}
+
+export interface WorkspacePreprocessingRouteDeps {
+  repository: CatalogRepository;
+  registry: WorkspaceRegistry;
+  service: Pick<WorkspacePreprocessingService, "run" | "clear">;
+  inputFingerprint?: Pick<ThtRunner, "workspaceInputFingerprint">;
+  readLatestJob?: (workspaceId: string) => PreprocessingJobState | undefined;
+}
+
+const PROGRESS_DETAILS = {
+  catalog_snapshot: "Preparing the PostgreSQL Catalog snapshot.",
+  schema_index: "Building schema vectors and LSH indexes.",
+  evidence: "Indexing Evidence.",
+  finalizing: "Publishing the completed preprocessing state.",
+} as const;
+
+function runningProgress(
+  workspaceId: string,
+  deps: WorkspacePreprocessingRouteDeps,
+): NonNullable<WorkspacePreprocessingStatus["progress"]> {
+  let job: PreprocessingJobState | undefined;
+  try {
+    job = deps.readLatestJob?.(workspaceId);
+  } catch {
+    // Progress is supplemental. A damaged or temporarily unavailable checkpoint must not hide
+    // the authoritative running state held by PostgreSQL.
+  }
+  const completed = new Set(job?.status === "active" ? job.completedStages : []);
+  if (completed.has("evidence")) return { stage: "finalizing", step: 4, totalSteps: 4 };
+  if (completed.has("schema_index")) return { stage: "evidence", step: 3, totalSteps: 4 };
+  if (completed.has("catalog_snapshot")) return { stage: "schema_index", step: 2, totalSteps: 4 };
+  return { stage: "catalog_snapshot", step: 1, totalSteps: 4 };
+}
+
+function base(
+  workspaceId: string,
+  state: WorkspacePreprocessingUiState,
+  detail: string,
+  database?: WorkspaceDatabase,
+): WorkspacePreprocessingStatus {
+  return {
+    schemaVersion: 1,
+    workspaceId,
+    state,
+    actionable: state === "ready" || state === "required" || state === "failed",
+    clearable: Boolean(
+      database
+      && state !== "running"
+      && database.preprocessingErrorCode !== "derived_data_cleared"
+    ),
+    detail,
+    ...(database ? {
+      metadataRevision: database.metadataContentRevision,
+      ...(database.preprocessedMetadataRevision === undefined
+        ? {}
+        : { preprocessedMetadataRevision: database.preprocessedMetadataRevision }),
+      ...(database.preprocessingStartedAt ? { startedAt: database.preprocessingStartedAt } : {}),
+      ...(database.preprocessingFinishedAt ? { finishedAt: database.preprocessingFinishedAt } : {}),
+    } : {}),
+  };
+}
+
+function blocked(
+  workspaceId: string,
+  detail: string,
+  reason: string,
+  nextStep: string,
+  database?: WorkspaceDatabase,
+): WorkspacePreprocessingStatus {
+  return { ...base(workspaceId, "blocked", detail, database), reason, nextStep };
+}
+
+function failureDiagnostic(errorCode: string): {
+  stage: string;
+  detail: string;
+  reason: string;
+  nextStep: string;
+} {
+  switch (errorCode) {
+    case "catalog_snapshot_failed":
+      return {
+        stage: "catalog_snapshot",
+        detail: "The Catalog snapshot could not be prepared.",
+        reason: "PostgreSQL Catalog metadata could not be read into a consistent preprocessing snapshot.",
+        nextStep: "Check Catalog availability, then retry preprocessing.",
+      };
+    case "schema_index_failed":
+      return {
+        stage: "schema_index",
+        detail: "The schema index could not be rebuilt.",
+        reason: "The schema indexing worker stopped before the Catalog snapshot was published to Qdrant.",
+        nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
+      };
+    case "evidence_preprocessing_failed":
+      return {
+        stage: "evidence",
+        detail: "Evidence preprocessing did not complete.",
+        reason: "The Evidence indexing worker stopped before it finished publishing the current workspace data.",
+        nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
+      };
+    case "semantic_index_incompatible":
+      return {
+        stage: "semantic_preflight",
+        detail: "The semantic index configuration is incompatible.",
+        reason: "Qdrant rejected the collection configuration for the active embedding model.",
+        nextStep: "Check embedding dimensions and Qdrant collection settings, then retry.",
+      };
+    case "egress_policy_refused":
+      return {
+        stage: "evidence",
+        detail: "The Evidence source was refused by policy.",
+        reason: "The configured Evidence endpoint is not allowed by the installation egress policy.",
+        nextStep: "Correct the Evidence source or its allowlist configuration, then retry.",
+      };
+    case "workspace_not_activatable":
+      return {
+        stage: "runtime_preflight",
+        detail: "The workspace runtime could not be prepared.",
+        reason: "The active workspace or one of its required runtime bindings is not usable.",
+        nextStep: "Check Workspace management and Database management, then retry.",
+      };
+    default:
+      return {
+        stage: "preprocessing",
+        detail: "Preprocessing did not complete.",
+        reason: "The preprocessing worker stopped before the current Catalog revision was published.",
+        nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
+      };
+  }
+}
+
+export async function readWorkspacePreprocessingStatus(
+  workspaceId: string,
+  deps: WorkspacePreprocessingRouteDeps,
+): Promise<WorkspacePreprocessingStatus> {
+  const database = await deps.repository.getByWorkspace(workspaceId);
+  if (!database) {
+    return blocked(
+      workspaceId,
+      "Database configuration is required.",
+      "This workspace has no database configuration in the PostgreSQL Catalog.",
+      "Open Database management and configure the workspace database.",
+    );
+  }
+
+  if (database.preprocessingStatus === "running") {
+    const progress = runningProgress(workspaceId, deps);
+    return {
+      ...base(workspaceId, "running", PROGRESS_DETAILS[progress.stage], database),
+      progress,
+    };
+  }
+  if (database.binding.transport === "ssh_tunnel") {
+    return blocked(
+      workspaceId,
+      "The database transport is not supported by the core runtime.",
+      "The current database binding uses an SSH tunnel, which cannot be used by a ThothII session.",
+      "Open Database management and select a supported runtime transport.",
+      database,
+    );
+  }
+  if (database.schemaSyncedVersion !== database.version) {
+    return blocked(
+      workspaceId,
+      "Catalog synchronization is required.",
+      `Database configuration v${database.version} is newer than the latest Catalog synchronization${database.schemaSyncedVersion === undefined ? "." : ` v${database.schemaSyncedVersion}.`}`,
+      "Open Database management and run Synchronize schema.",
+      database,
+    );
+  }
+
+  const [syncRuns, activeDescriptionRun, sensitivityRuns] = await Promise.all([
+    deps.repository.listSyncRuns(database.id, 10),
+    deps.repository.getActiveDescriptionGenerationRun(),
+    deps.repository.listSensitivityAnalysisRuns(50),
+  ]);
+  if (syncRuns.some((run) => ACTIVE_SYNC_STATES.has(run.state))) {
+    return blocked(
+      workspaceId,
+      "Catalog synchronization is in progress.",
+      "The Catalog is being synchronized and its metadata revision is not stable yet.",
+      "Wait for schema synchronization to finish, then run preprocessing.",
+      database,
+    );
+  }
+  if (activeDescriptionRun?.databaseId === database.id
+    && ["queued", "running"].includes(activeDescriptionRun.status)) {
+    return blocked(
+      workspaceId,
+      "Description generation is in progress.",
+      "Catalog descriptions are still being generated for this database.",
+      "Wait for description generation to finish, then run preprocessing.",
+      database,
+    );
+  }
+  if (sensitivityRuns.some((run) => run.databaseId === database.id && run.status === "running")) {
+    return blocked(
+      workspaceId,
+      "Sensitivity analysis is in progress.",
+      "Catalog sensitivity metadata is still being analyzed for this database.",
+      "Wait for sensitivity analysis to finish, then run preprocessing.",
+      database,
+    );
+  }
+
+  let inputFingerprint: string | undefined;
+  try {
+    const record = await deps.registry.read(workspaceId);
+    if (deps.inputFingerprint) {
+      inputFingerprint = await deps.inputFingerprint.workspaceInputFingerprint(
+        record.revision.snapshotPath,
+      );
+    }
+  } catch {
+    return blocked(
+      workspaceId,
+      "The workspace runtime configuration is unavailable.",
+      "The active workspace revision or one of its required database secrets could not be resolved.",
+      "Check Workspace management and Database management before running preprocessing.",
+      database,
+    );
+  }
+
+  const current = database.preprocessingStatus === "succeeded"
+    && database.preprocessedMetadataRevision === database.metadataContentRevision
+    && (inputFingerprint === undefined
+      || database.preprocessingInputFingerprint === inputFingerprint);
+  if (current) {
+    return base(
+      workspaceId,
+      "ready",
+      `Catalog revision ${database.metadataContentRevision} is indexed.`,
+      database,
+    );
+  }
+
+  if (database.preprocessingErrorCode === "derived_data_cleared") {
+    return base(
+      workspaceId,
+      "required",
+      "Reference vectors and LSH are empty. Memory is preserved.",
+      database,
+    );
+  }
+
+  if (database.preprocessingStatus === "failed"
+    && database.preprocessingErrorCode
+    && database.preprocessingErrorCode !== "catalog_changed"
+    && database.preprocessingErrorCode !== "derived_data_cleared"
+    && database.preprocessingFinishedAt) {
+    const diagnostic = failureDiagnostic(database.preprocessingErrorCode);
+    return {
+      ...base(workspaceId, "failed", diagnostic.detail, database),
+      reason: diagnostic.reason,
+      nextStep: diagnostic.nextStep,
+      lastFailure: {
+        stage: diagnostic.stage,
+        errorCode: database.preprocessingErrorCode,
+        finishedAt: database.preprocessingFinishedAt,
+      },
+    };
+  }
+
+  return base(
+    workspaceId,
+    "required",
+    `Catalog revision ${database.metadataContentRevision} is not indexed.`,
+    database,
+  );
+}
+
+function safeError(reply: FastifyReply, error: unknown) {
+  if (error instanceof CatalogUnavailableError) {
+    return reply.code(503).send({
+      code: "catalog_unavailable",
+      message: "Database catalog is unavailable.",
+    });
+  }
+  if (error instanceof z.ZodError) {
+    return reply.code(400).send({
+      code: "preprocessing_request_invalid",
+      message: "Preprocessing request is invalid.",
+    });
+  }
+  return reply.code(500).send({
+    code: "preprocessing_run_failed",
+    message: "Preprocessing status could not be resolved.",
+  });
+}
+
+function workspaceIdFrom(request: FastifyRequest): string {
+  return workspaceIdSchema.parse((request.params as { workspaceId?: unknown }).workspaceId);
+}
+
+export function workspacePreprocessingRoutes(
+  app: FastifyInstance,
+  deps: WorkspacePreprocessingRouteDeps,
+): void {
+  app.get("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
+    if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply;
+    try {
+      return await readWorkspacePreprocessingStatus(workspaceIdFrom(request), deps);
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+
+  app.post("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
+    if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
+    try {
+      const workspaceId = workspaceIdFrom(request);
+      const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
+      if (!before.actionable) {
+        return reply.code(409).send({ ...before, code: "preprocessing_blocked" });
+      }
+
+      const result = await deps.service.run({ workspaceId });
+      const after = await readWorkspacePreprocessingStatus(workspaceId, deps);
+      if (after.state === "ready") return after;
+      if (after.state === "failed") {
+        return reply.code(422).send({ ...after, code: "preprocessing_run_failed" });
+      }
+      if (after.state === "blocked" || after.state === "running") {
+        return reply.code(409).send({ ...after, code: "preprocessing_blocked" });
+      }
+      return reply.code(500).send({
+        code: "preprocessing_run_failed",
+        message: `Preprocessing ended with ${result.code}.`,
+      });
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+
+  app.delete("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
+    if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
+    try {
+      const workspaceId = workspaceIdFrom(request);
+      const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
+      if (!before.clearable) {
+        return reply.code(409).send({ ...before, code: "preprocessing_clear_blocked" });
+      }
+      const result = await deps.service.clear({ workspaceId });
+      if (result.status !== "succeeded") {
+        return reply.code(result.code === "preprocessing_conflict" ? 409 : 500).send({
+          code: result.code,
+          message: "Preprocessing data could not be cleared.",
+        });
+      }
+      return await readWorkspacePreprocessingStatus(workspaceId, deps);
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+}

backend/src/routes/workspaces.ts

@@ -16,23 +16,33 @@ import {
   type WorkspaceDescriptor,
 } from "../workspaces/schema.js";
 import type { RuntimeBindings } from "../workspaces/runtime-renderer.js";
-import type { ConnectorDiagnostics } from "../workspaces/diagnostics.js";
+import type {
+  ConnectorDiagnostics,
+  Diagnostic,
+  WorkspaceDiagnosticOptions,
+} from "../workspaces/diagnostics.js";
 import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
 import type { AuthDiagnoser } from "../auth/diagnostics.js";
 import { decodeAuthDiagnostics, type AuthDiagnostics } from "../auth/group-catalog.js";
+import type { WorkspaceDatabase } from "../catalog/types.js";
 
 export type WorkspaceDiagnoser = (
   workspace: WorkspaceDescriptor,
   bindings: RuntimeBindings,
-  options: { writeProbe: boolean },
+  options: WorkspaceDiagnosticOptions,
 ) => Promise<ConnectorDiagnostics>;
 
+export type WorkspaceDatabaseTester = (
+  workspaceId: string,
+) => Promise<WorkspaceDatabase | undefined>;
+
 interface WorkspaceRoutesDeps {
   registry: WorkspaceRegistry;
   config: WorkspaceRegistryConfig;
   diagnose: WorkspaceDiagnoser;
   authDiagnoser: AuthDiagnoser;
   secretStore: WorkspaceSecretStore;
+  testDatabaseConnection: WorkspaceDatabaseTester;
 }
 
 const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
@@ -56,6 +66,20 @@ const SAFE_MESSAGES = {
   semantic_index_incompatible: "Semantic index is incompatible with this workspace.",
 } as const;
 
+const catalogConnectionUnavailable = (): Diagnostic => ({
+  level: "error",
+  code: "connector_unavailable",
+  field: "dwh",
+  message: "The configured database could not be reached or authenticated.",
+});
+
+const catalogConnectionMissing = (): Diagnostic => ({
+  level: "error",
+  code: "binding_missing",
+  field: "dwh",
+  message: "Configure this workspace in Database Management before testing connections.",
+});
+
 function authenticationReport(value: unknown): AuthDiagnostics {
   const report = decodeAuthDiagnostics(value);
   if (!report) throw new Error("invalid authentication diagnostic report");
@@ -238,14 +262,33 @@ export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps)
         deps.secretStore,
       );
       try {
-        const [workspaceDiagnostics, inspectedAuthentication] = await Promise.all([
-          deps.diagnose(operational, lease.bindings, { writeProbe: false }),
+        const [workspaceDiagnostics, testedDatabase, inspectedAuthentication] = await Promise.all([
+          deps.diagnose(operational, lease.bindings, {
+            writeProbe: false,
+            skipDwh: true,
+          }),
+          deps.testDatabaseConnection(id),
           deps.authDiagnoser.inspect({ live: true }),
         ]);
         const authentication = authenticationReport(inspectedAuthentication);
+        const catalogConnectionReady = testedDatabase?.connectionStatus === "reachable";
+        const catalogConnectionDiagnostic = !testedDatabase
+          ? catalogConnectionMissing()
+          : catalogConnectionReady
+            ? undefined
+            : catalogConnectionUnavailable();
+        const diagnostics = catalogConnectionDiagnostic
+          ? [
+            ...workspaceDiagnostics.diagnostics.filter(({ code }) => code !== "binding_ok"),
+            catalogConnectionDiagnostic,
+          ]
+          : workspaceDiagnostics.diagnostics;
         return {
           ...workspaceDiagnostics,
-          activatable: workspaceDiagnostics.activatable && authentication.ready,
+          activatable: workspaceDiagnostics.activatable
+            && catalogConnectionReady
+            && authentication.ready,
+          diagnostics,
           authentication,
         };
       } finally {

backend/src/settings/settings-store.ts

@@ -13,6 +13,7 @@ import type { AppConfig } from "../config.js";
 
 export interface Settings {
   workspace?: string;
+  /** Legacy input fields are ignored when loading/evaluating installation settings. */
   provider?: string;
   model?: string;
   thinking?: string;
@@ -37,7 +38,13 @@ export function loadSettings(cfg: AppConfig): Settings {
   try {
     const raw = readFileSync(cfg.settingsFile, "utf8");
     const parsed = JSON.parse(raw);
-    if (parsed && typeof parsed === "object") return parsed as Settings;
+    if (parsed && typeof parsed === "object") {
+      const value = parsed as Record<string, unknown>;
+      return {
+        ...(typeof value.workspace === "string" ? { workspace: value.workspace } : {}),
+        ...(typeof value.thinking === "string" ? { thinking: value.thinking } : {}),
+      };
+    }
     return {};
   } catch {
     return {};

backend/src/tht/interaction-language.ts

@@ -0,0 +1,10 @@
+/** Validate/canonicalize the tag; available translation catalogs belong to the UI. */
+export function interactionLanguage(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined;
+  try {
+    const [canonical] = Intl.getCanonicalLocales(value);
+    return canonical;
+  } catch {
+    return undefined;
+  }
+}

backend/src/tht/tht-runner.ts

@@ -5,7 +5,7 @@ import {
   openSync, readFileSync, readSync, realpathSync, statSync, unlinkSync, writeFileSync,
 } from "node:fs";
 import { dirname, isAbsolute, join, relative, resolve } from "node:path";
-import { parseAllDocuments } from "yaml";
+import { parse, parseAllDocuments } from "yaml";
 import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js";
 import { secretValue, type SecretBundleConfig } from "../config/secret-bundle.js";
 import { renderWorkspaceRuntimeFromSnapshotPath } from "../workspaces/runtime-config-lease.js";
@@ -22,6 +22,9 @@ import {
 } from "../workspaces/schema.js";
 import { reconcileCollection, type CollectionMode } from "../workspaces/qdrant-collection.js";
 import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
+import type { CatalogRepository } from "../catalog/types.js";
+import { preprocessingInputFingerprint } from "../workspaces/effective-config.js";
+import { workspaceVectorCollections } from "../workspaces/vector-collections.js";
 
 export interface ThtConfig extends SecretBundleConfig {
   thtBin: string;
@@ -35,12 +38,14 @@ export interface ThtConfig extends SecretBundleConfig {
   /** "self_heal" for session admission (create missing collections/indexes), default "require_existing". */
   qdrantCollectionMode?: "self_heal" | "require_existing";
   workspaceSecretStore?: WorkspaceSecretStore;
+  catalogRepository?: CatalogRepository;
 }
 
 export interface RuntimeConfigLease {
   path: string;
   workspaceId: string;
   workspaceRevision: string;
+  inputFingerprint: string;
   release(): void;
 }
 
@@ -54,6 +59,7 @@ export interface SessionRow {
   author: string | null;
   workspace_id?: string | null;
   workspace_revision?: string | null;
+  interaction_language?: string | null;
   archived?: boolean;
 }
 
@@ -79,6 +85,7 @@ export type SemanticReadinessCode = "workspace_not_activatable" | "semantic_inde
 export interface QdrantEnsureResult {
   ok: boolean;
   code?: SemanticReadinessCode;
+  state?: "ready" | "created" | "repaired" | "upgraded";
 }
 
 const REQUIRED_QDRANT_PAYLOAD_INDEXES = [
@@ -213,37 +220,31 @@ export class ThtRunner {
   }
 
   /** Render one immutable canonical registry revision into a backend-owned harness config. */
-  acquireWorkspaceRuntime(
-    workspaceConfigPath: string,
-    effectiveRelationships?: string,
-  ): RuntimeConfigLease {
-    const effectiveRelationshipsPath = effectiveRelationships === undefined
+  async acquireWorkspaceRuntime(workspaceConfigPath: string): Promise<RuntimeConfigLease> {
+    const identity = this.assertWorkspaceSnapshot(workspaceConfigPath);
+    const catalogDatabase = this.cfg.catalogRepository === undefined
       ? undefined
-      : this.createRuntimeSnapshot(effectiveRelationships);
-    let rendered: ReturnType<typeof renderWorkspaceRuntimeFromSnapshotPath>;
-    try {
-      rendered = renderWorkspaceRuntimeFromSnapshotPath({
-        snapshotPath: workspaceConfigPath,
-        harnessDir: this.cfg.harnessDir,
-        configPath: this.cfg.configPath,
-        dataRoot: this.cfg.dataRoot ?? (() => {
-          throw new Error("registry workspace runtime requires an absolute data root");
-        })(),
-        secretRoots: this.cfg.secretRoots ?? [],
-        semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
-        workspaceSecretStore: this.cfg.workspaceSecretStore,
-        effectiveRelationshipsPath,
-      });
-    } catch (error) {
-      if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
-      throw error;
+      : await this.cfg.catalogRepository.getByWorkspace(identity.workspaceId);
+    if (this.cfg.catalogRepository !== undefined && !catalogDatabase) {
+      throw new Error("workspace database is not configured in the Catalog");
     }
+    const rendered = renderWorkspaceRuntimeFromSnapshotPath({
+      snapshotPath: workspaceConfigPath,
+      harnessDir: this.cfg.harnessDir,
+      configPath: this.cfg.configPath,
+      dataRoot: this.cfg.dataRoot ?? (() => {
+        throw new Error("registry workspace runtime requires an absolute data root");
+      })(),
+      secretRoots: this.cfg.secretRoots ?? [],
+      semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
+      workspaceSecretStore: this.cfg.workspaceSecretStore,
+      catalogDatabase,
+    });
     let path: string;
     try {
       path = this.createRuntimeSnapshot(rendered.renderedConfig);
     } catch (error) {
       rendered.releaseSecrets();
-      if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
       throw error;
     }
     let released = false;
@@ -251,16 +252,29 @@ export class ThtRunner {
       path,
       workspaceId: rendered.workspaceId,
       workspaceRevision: rendered.workspaceRevision,
+      inputFingerprint: preprocessingInputFingerprint(
+        rendered.workspaceId,
+        rendered.workspaceRevision,
+        parse(rendered.renderedConfig),
+      ),
       release: () => {
         if (released) return;
         released = true;
         this.cleanupRuntimeSnapshot(path);
-        if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
         rendered.releaseSecrets();
       },
     };
   }
 
+  async workspaceInputFingerprint(workspaceConfigPath: string): Promise<string> {
+    const lease = await this.acquireWorkspaceRuntime(workspaceConfigPath);
+    try {
+      return lease.inputFingerprint;
+    } finally {
+      lease.release();
+    }
+  }
+
   private runtimeSnapshotDirectory(): string {
     if (!this.cfg.runtimeSnapshotRoot) throw new Error("runtime snapshot root is not configured");
     if (!isAbsolute(this.cfg.runtimeSnapshotRoot)) throw new Error("runtime snapshot root must be absolute");
@@ -392,13 +406,9 @@ export class ThtRunner {
       workspaceConfigPath && isAbsolute(workspaceConfigPath)
       && !this.runtimeSnapshots.has(workspaceConfigPath)
     ) {
-      let runtime: RuntimeConfigLease;
-      try {
-        runtime = this.acquireWorkspaceRuntime(workspaceConfigPath);
-      } catch (error) {
-        return Promise.reject(error);
-      }
-      return this.run(args, runtime.path, timeoutMs).finally(runtime.release);
+      return this.acquireWorkspaceRuntime(workspaceConfigPath).then((runtime) => (
+        this.run(args, runtime.path, timeoutMs).finally(runtime.release)
+      ));
     }
     return new Promise((resolve) => {
       const env: NodeJS.ProcessEnv = { ...process.env };
@@ -473,6 +483,7 @@ export class ThtRunner {
 
   async sessionNew(o: {
     question: string;
+    interactionLanguage?: string;
     provider?: string;
     model?: string;
     thinking?: string;
@@ -488,6 +499,7 @@ export class ThtRunner {
       ["--provider", o.provider],
       ["--model", o.model],
       ["--thinking", o.thinking],
+      ["--interaction-language", o.interactionLanguage],
       ["--name", o.name],
       ["--workspace-id", o.workspaceId],
       ["--workspace-revision", o.workspaceRevision],
@@ -495,7 +507,7 @@ export class ThtRunner {
       if (v) a.push(f, v);
     }
     a.push("--json");
-    return this.json<{ id: string }>(a, o.workspaceConfigPath ?? o.workspace);
+    return this.json<{ id: string; interaction_language?: string }>(a, o.workspaceConfigPath ?? o.workspace);
   }
 
   /** Build and persist the deterministic F1 retrieval pack for a new session. */
@@ -524,6 +536,12 @@ export class ThtRunner {
     return this.json<unknown>(["session", "show", id, "--json"], workspace);
   }
 
+  ensureInteractionLanguage(id: string, workspace?: string) {
+    return this.json<{ interaction_language: string }>(
+      ["session", "ensure-interaction-language", id, "--json"], workspace,
+    );
+  }
+
   sqlPreview(id: string, p: { limit?: number; offset?: number }, workspace?: string) {
     // No positional FILE: the harness resolves sql_final.sql from the session
     // via _session_sql_file(cfg, session_id), which respects the workspace path.
@@ -598,20 +616,26 @@ export class ThtRunner {
     } catch {
       return { ok: false, code: "workspace_not_activatable" };
     }
-    const collection = descriptor.semantic_index.vector_store;
+    const collections = workspaceVectorCollections(descriptor.workspace.id);
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutSec) * 1000);
     try {
-      const checked = await reconcileCollection({
-        baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
-        collection: collection.collection,
-        dimensions: collection.dimensions,
-        distance: collection.distance,
-        mode,
-        request: this.cfg.qdrantRequest ?? fetch,
-        signal: controller.signal,
-      });
-      return checked;
+      const checked = await Promise.all(Object.entries(collections).map(async ([purpose, collection]) =>
+        await reconcileCollection({
+          baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
+          collection,
+          dimensions: this.cfg.semanticRuntime.internalEmbeddingDimensions,
+          distance: "cosine",
+          mode: mode === "evidence_maintenance" && purpose === "memory" ? "self_heal" : mode,
+          request: this.cfg.qdrantRequest ?? fetch,
+          signal: controller.signal,
+        })));
+      if (!checked.every((result) => result.ok)) {
+        return { ok: false, code: "semantic_index_incompatible" };
+      }
+      const state = (["upgraded", "repaired", "created", "ready"] as const)
+        .find((candidate) => checked.some((result) => result.state === candidate));
+      return { ok: true, ...(state ? { state } : {}) };
     } catch {
       return { ok: false, code: "workspace_not_activatable" };
     } finally {

backend/src/workspace-maintenance.ts

@@ -1,15 +1,14 @@
 import { spawn } from "node:child_process";
 import { closeSync, constants as fsConstants, openSync } from "node:fs";
-import { readdir, readFile } from "node:fs/promises";
 import { join } from "node:path";
-import { parse } from "yaml";
-import { loadConfig } from "./config.js";
+import { loadConfig, type AppConfig } from "./config.js";
 import { ThtRunner } from "./tht/tht-runner.js";
 import { WorkspaceRegistry } from "./workspaces/registry.js";
 import { publishDeterministicRuntimeConfigLease, renderActiveWorkspaceRuntime } from "./workspaces/runtime-config-lease.js";
 import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
 import { WorkspacePreprocessingService, type WorkspaceOperationResult } from "./workspaces/preprocessing-service.js";
-import type { SessionInventoryRow } from "./workspaces/preprocessing-state.js";
+import { createCatalogRepository } from "./catalog/repository.js";
+import type { CatalogRepository } from "./catalog/types.js";
 
 export interface WorkspaceMaintenanceIo {
   stdin: string;
@@ -19,7 +18,7 @@ export interface WorkspaceMaintenanceIo {
   writeStderr(value: string): void;
 }
 
-type Command = "inspect" | "preprocess-dwh" | "schema-suggest-fks" | "schema-check" | "schema-accept" | "index-schema" | "preprocess-evidence" | "preprocess-run" | "vector-inspect" | "vector-rebuild";
+type Command = "inspect" | "preprocess-run" | "preprocess-clear" | "evidence-consolidate" | "evidence-refresh" | "evidence-decide";
 
 function failureResult(
   operation: string,
@@ -64,15 +63,11 @@ function parseRequest(command: string, stdin: string): Record<string, unknown> {
   }
   const allowedByCommand: Record<string, readonly string[]> = {
     inspect: ["schemaVersion", "workspaceId"],
-    "preprocess-dwh": ["schemaVersion", "workspaceId", "resumeRunId"],
-    "schema-suggest-fks": ["schemaVersion", "workspaceId", "fromSql", "assume", "resumeRunId"],
-    "schema-check": ["schemaVersion", "workspaceId", "annotationsYaml", "reviewedCandidatesDigest"],
-    "schema-accept": ["schemaVersion", "workspaceId", "runId", "yes"],
-    "index-schema": ["schemaVersion", "workspaceId", "resumeRunId"],
-    "preprocess-evidence": ["schemaVersion", "workspaceId", "dryRun", "resumeRunId"],
-    "preprocess-run": ["schemaVersion", "workspaceId", "resumeRunId"],
-    "vector-inspect": ["schemaVersion", "workspaceId"],
-    "vector-rebuild": ["schemaVersion", "workspaceId", "collection", "confirm", "destroy"],
+    "preprocess-run": ["schemaVersion", "workspaceId"],
+    "preprocess-clear": ["schemaVersion", "workspaceId"],
+    "evidence-consolidate": ["schemaVersion", "workspaceId"],
+    "evidence-refresh": ["schemaVersion", "workspaceId"],
+    "evidence-decide": ["schemaVersion", "workspaceId", "sourceId", "revision", "decision"],
   };
   const allowed = allowedByCommand[command];
   if (!allowed) throw new Error("unknown command");
@@ -89,57 +84,24 @@ function exitCodeFor(result: WorkspaceOperationResult): number {
 
 async function dispatch(command: Command, service: WorkspacePreprocessingService, request: Record<string, unknown>): Promise<WorkspaceOperationResult> {
   switch (command) {
+    case "evidence-refresh":
+      return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "refresh" });
+    case "evidence-decide":
+      if (typeof request.sourceId !== "string" || !/^[a-f0-9]{64}$/.test(request.sourceId)
+        || typeof request.revision !== "string" || !/^[a-f0-9]{64}$/.test(request.revision)
+        || (request.decision !== "keep" && request.decision !== "replace")) throw new Error("invalid request");
+      return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "decide",
+        sourceId: request.sourceId, revision: request.revision, decision: request.decision });
+    case "evidence-consolidate":
+      return await service.consolidateEvidence({ workspaceId: request.workspaceId as string });
     case "inspect":
       return await service.inspect({ workspaceId: request.workspaceId as string });
-    case "preprocess-dwh":
-      return await service.preprocessDwh({
-        workspaceId: request.workspaceId as string,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "schema-suggest-fks":
-      return await service.suggestFks({
-        workspaceId: request.workspaceId as string,
-        fromSql: request.fromSql as any,
-        assume: request.assume as any,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "schema-check":
-      return await service.checkSchema({
-        workspaceId: request.workspaceId as string,
-        annotationsYaml: request.annotationsYaml as string | undefined,
-        reviewedCandidatesDigest: request.reviewedCandidatesDigest as string | undefined,
-      });
-    case "schema-accept":
-      return await service.acceptSchema({
-        workspaceId: request.workspaceId as string,
-        runId: request.runId as string,
-        yes: request.yes === true,
-      });
-    case "index-schema":
-      return await service.indexSchema({
-        workspaceId: request.workspaceId as string,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "preprocess-evidence":
-      return await service.preprocessEvidence({
-        workspaceId: request.workspaceId as string,
-        dryRun: request.dryRun as boolean | undefined,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
     case "preprocess-run":
       return await service.run({
         workspaceId: request.workspaceId as string,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "vector-inspect":
-      return await service.vectorInspect({ workspaceId: request.workspaceId as string });
-    case "vector-rebuild":
-      return await service.vectorRebuild({
-        workspaceId: request.workspaceId as string,
-        collection: request.collection as string | undefined,
-        confirm: request.confirm as string | undefined,
-        destroy: request.destroy === true,
       });
+    case "preprocess-clear":
+      return await service.clear({ workspaceId: request.workspaceId as string });
   }
 }
 
@@ -178,48 +140,32 @@ export async function runWorkspaceMaintenanceCli(
       || message === "unexpected request field"
       || message === "invalid workspace id";
     return command in {
-      inspect: true, "preprocess-dwh": true, "schema-suggest-fks": true, "schema-check": true,
-      "schema-accept": true,
-      "index-schema": true, "preprocess-evidence": true, "preprocess-run": true,
+      inspect: true, "preprocess-run": true, "preprocess-clear": true, "evidence-consolidate": true,
+      "evidence-refresh": true, "evidence-decide": true,
     } ? (requestError ? 2 : 1) : 2;
   }
 }
 
-async function readSessionInventory(dataRoot: string, workspaceId: string): Promise<readonly SessionInventoryRow[]> {
-  const directory = join(dataRoot, "sessions", workspaceId, "sessions");
-  try {
-    const entries = await readdir(directory, { withFileTypes: true });
-    const rows: SessionInventoryRow[] = [];
-    for (const entry of entries) {
-      if (!entry.isDirectory() || entry.isSymbolicLink()) continue;
-      try {
-        const source = await readFile(join(directory, entry.name, "session_manifest.yaml"), "utf8");
-        const manifest = parse(source) as Record<string, unknown>;
-        rows.push({
-          id: entry.name,
-          status: typeof manifest.status === "string" ? manifest.status : "open",
-          archived: manifest.archived === true,
-          workspaceRevision: typeof manifest.workspace_revision === "string" ? manifest.workspace_revision : null,
-        });
-      } catch {
-        // fail closed at mutation time by ignoring unreadable manifests from the resumable scan
-      }
-    }
-    return rows;
-  } catch {
-    return [];
-  }
+export interface ProductionWorkspacePreprocessingDeps {
+  config?: AppConfig;
+  catalogRepository?: CatalogRepository;
+  registry?: WorkspaceRegistry;
+  workspaceSecretStore?: WorkspaceSecretStore;
+  runner?: ThtRunner;
 }
 
-function createProductionService(): WorkspacePreprocessingService {
-  const config = loadConfig(process.env, { surface: "workspace-maintenance" });
-  const registry = new WorkspaceRegistry(config.workspaceRegistry);
-  const workspaceSecretStore = new WorkspaceSecretStore({
+export function createProductionWorkspacePreprocessingService(
+  deps: ProductionWorkspacePreprocessingDeps = {},
+): WorkspacePreprocessingService {
+  const config = deps.config ?? loadConfig(process.env, { surface: "workspace-maintenance" });
+  const catalogRepository = deps.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
+  const registry = deps.registry ?? new WorkspaceRegistry(config.workspaceRegistry);
+  const workspaceSecretStore = deps.workspaceSecretStore ?? new WorkspaceSecretStore({
     root: config.workspaceSecretStoreRoot,
     runtimeRoot: config.workspaceSecretRuntimeRoot,
     installationId: config.workspaceRegistry.installationId,
   });
-  const runner = new ThtRunner({
+  const runner = deps.runner ?? new ThtRunner({
     thtBin: config.thtBin,
     harnessDir: config.harnessDir,
     configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
@@ -232,6 +178,7 @@ function createProductionService(): WorkspacePreprocessingService {
     semanticRuntime: {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
+      internalEmbeddingId: config.internalEmbeddingId,
       internalEmbeddingModel: config.internalEmbeddingModel,
       internalEmbeddingDimensions: config.internalEmbeddingDimensions,
     },
@@ -240,7 +187,10 @@ function createProductionService(): WorkspacePreprocessingService {
     dataRoot: config.dataRoot ?? "/data",
     httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "")
       .split(",").map((value) => value.trim()).filter((value) => value.length > 0),
+    catalogRepository,
     acquireActiveRuntime: async (workspaceId) => {
+      const catalogDatabase = await catalogRepository.getByWorkspace(workspaceId);
+      if (!catalogDatabase) throw new Error("workspace database is not configured in the Catalog");
       const active = await renderActiveWorkspaceRuntime({
         workspaceId,
         registry,
@@ -250,6 +200,7 @@ function createProductionService(): WorkspacePreprocessingService {
         dataRoot: config.dataRoot ?? "/data",
         secretRoots: config.workspaceRegistry.secretRoots,
         workspaceSecretStore,
+        catalogDatabase,
         semanticRuntime: {
           internalQdrantUrl: config.internalQdrantUrl,
           internalEmbeddingUrl: config.internalEmbeddingUrl,
@@ -257,38 +208,55 @@ function createProductionService(): WorkspacePreprocessingService {
           internalEmbeddingDimensions: config.internalEmbeddingDimensions,
         },
       });
-      const configLease = await publishDeterministicRuntimeConfigLease({
-        workspaceId,
-        registry,
-        registryConfig: config.workspaceRegistry,
-        harnessDir: config.harnessDir,
-        configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
-        dataRoot: config.dataRoot ?? "/data",
-        secretRoots: config.workspaceRegistry.secretRoots,
-        semanticRuntime: {
-          internalQdrantUrl: config.internalQdrantUrl,
-          internalEmbeddingUrl: config.internalEmbeddingUrl,
-          internalEmbeddingModel: config.internalEmbeddingModel,
-          internalEmbeddingDimensions: config.internalEmbeddingDimensions,
-        },
-        workspaceSecretStore,
-      });
-      return {
-        workspace: active.workspace,
-        workspaceId: active.workspaceId,
-        workspaceRevision: active.workspaceRevision,
-        descriptorBlob: active.descriptorBlob,
-        catalogBlob: active.catalogBlob,
-        configLease,
-      };
+      try {
+        const configLease = await publishDeterministicRuntimeConfigLease({
+          workspaceId,
+          registry,
+          registryConfig: config.workspaceRegistry,
+          harnessDir: config.harnessDir,
+          configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
+          dataRoot: config.dataRoot ?? "/data",
+          secretRoots: config.workspaceRegistry.secretRoots,
+          semanticRuntime: {
+            internalQdrantUrl: config.internalQdrantUrl,
+            internalEmbeddingUrl: config.internalEmbeddingUrl,
+            internalEmbeddingModel: config.internalEmbeddingModel,
+            internalEmbeddingDimensions: config.internalEmbeddingDimensions,
+          },
+          workspaceSecretStore,
+          catalogDatabase,
+        });
+        return {
+          workspace: active.workspace,
+          workspaceId: active.workspaceId,
+          workspaceRevision: active.workspaceRevision,
+          descriptorBlob: active.descriptorBlob,
+          catalogBlob: active.catalogBlob,
+          configLease,
+        };
+      } finally {
+        active.releaseSecrets();
+      }
     },
     runChild: async ({ argv, configPath }) => {
       const configFd = openSync(configPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
       try {
         return await new Promise((resolve) => {
+          const childEnvironment = { ...process.env };
+          for (const name of [
+            "THT_CATALOG_DATABASE_URL",
+            "THT_CATALOG_DB_HOST",
+            "THT_CATALOG_DB_PORT",
+            "THT_CATALOG_DB_NAME",
+            "THT_CATALOG_RUNTIME_USER",
+            "THT_CATALOG_RUNTIME_PASSWORD_FILE",
+            "THT_CATALOG_MIGRATOR_DATABASE_URL",
+            "THT_CATALOG_MIGRATOR_USER",
+            "THT_CATALOG_MIGRATOR_PASSWORD_FILE",
+          ]) delete childEnvironment[name];
           const child = spawn(config.thtBin, argv, {
             cwd: config.harnessDir,
-            env: { ...process.env, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
+            env: { ...childEnvironment, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
             stdio: ["ignore", "pipe", "pipe", configFd],
           });
           let stdout = "";
@@ -302,9 +270,8 @@ function createProductionService(): WorkspacePreprocessingService {
         closeSync(configFd);
       }
     },
-    listSessions: async (workspaceId) => await readSessionInventory(config.dataRoot ?? "/data", workspaceId),
     semanticPreflight: async (workspace) => {
-      const result = await runner.qdrantEnsure(workspace, 30);
+      const result = await runner.qdrantEnsure(workspace, 30, "self_heal");
       return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" };
     },
     evidencePreflight: async (workspace) => {
@@ -329,7 +296,11 @@ if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).
     writeStdout: (value) => { stdout.push(value); },
     writeStderr: (value) => { stderr.push(value); },
   };
-  const exitCode = await runWorkspaceMaintenanceCli(process.argv, createProductionService(), io);
+  const exitCode = await runWorkspaceMaintenanceCli(
+    process.argv,
+    createProductionWorkspacePreprocessingService(),
+    io,
+  );
   process.stdout.write(stdout.join(""));
   if (stderr.length > 0) process.stderr.write(stderr.join("").slice(0, 64 * 1024));
   process.exit(exitCode);

backend/src/workspaces/bindings.ts

@@ -59,12 +59,12 @@ function safeSecretFilePath(path: string, secretRoots: readonly string[]): strin
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Workspace bindings support only workspace schema version 3");
+    throw new Error("Workspace bindings support only workspace schema version 4");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Workspace bindings support only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Workspace bindings support only workspace schema version 4");
   }
 }
 
@@ -90,6 +90,7 @@ export function resolveBinding(
 ): ResolvedBinding {
   requireSupportedDescriptor(workspace);
   const descriptor = validateWorkspaceDescriptor(workspace);
+  if (!descriptor.dwh) throw new Error("workspace database is not bound in the descriptor");
   const contract = buildInstallationContract(descriptor);
   const variables = contract.variables.filter((variable) => variable.role === role);
   const transportVariable = variables.find((variable) => variable.suffix === "TRANSPORT");
@@ -155,7 +156,7 @@ export function resolveEvidenceBinding(
   return { values, missing };
 }
 
-/** Resolve the complete schema-v3 runtime binding set. */
+/** Resolve the complete schema-v4 runtime binding set. */
 export function resolveRuntimeBindings(
   workspace: WorkspaceDescriptor,
   env: NodeJS.ProcessEnv,
@@ -165,7 +166,9 @@ export function resolveRuntimeBindings(
   const descriptor = validateWorkspaceDescriptor(workspace);
 
   return {
-    dwh: resolveBinding(descriptor, "DWH", env, secretRoots),
+    dwh: descriptor.dwh
+      ? resolveBinding(descriptor, "DWH", env, secretRoots)
+      : { transport: "postgres_direct", values: {}, missing: [] },
     evidence: resolveEvidenceBinding(descriptor, env, secretRoots),
   };
 }

backend/src/workspaces/contracts.ts

@@ -137,12 +137,12 @@ function evidenceVariables(
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Installation contract supports only workspace schema version 3");
+    throw new Error("Installation contract supports only workspace schema version 4");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Installation contract supports only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Installation contract supports only workspace schema version 4");
   }
 }
 
@@ -155,7 +155,9 @@ export function buildInstallationContract(workspace: WorkspaceDescriptor): Insta
     workspaceId: descriptor.workspace.id,
     namespace,
     variables: [
-      ...connectorVariables(namespace, descriptor.dwh.supported_transports),
+      ...(descriptor.dwh
+        ? connectorVariables(namespace, descriptor.dwh.supported_transports)
+        : []),
       ...evidenceVariables(namespace, descriptor),
     ],
   };

backend/src/workspaces/diagnostics.ts

@@ -12,6 +12,7 @@ import {
 import type { WorkspaceErrorCode } from "./types.js";
 import type { SemanticRuntimeConfig } from "./runtime-renderer.js";
 import type { AuthDiagnostics } from "../auth/diagnostics.js";
+import { workspaceVectorCollections } from "./vector-collections.js";
 
 export interface Diagnostic {
   level: "error" | "warning" | "info";
@@ -130,6 +131,11 @@ export interface DiagnosticAdapters {
   probeEmbedding(request: EmbeddingDiagnosticRequest): Promise<EmbeddingDiagnosticResult>;
 }
 
+export interface WorkspaceDiagnosticOptions {
+  writeProbe: boolean;
+  skipDwh?: boolean;
+}
+
 export const DEFAULT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS = 5_000;
 
 async function secretPresent(file: string): Promise<boolean> {
@@ -406,12 +412,12 @@ function numericBinding(binding: Record<string, string>, name: string): number |
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Workspace diagnoser supports only workspace schema version 3");
+    throw new Error("Workspace diagnoser supports only workspace schema version 4");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Workspace diagnoser supports only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Workspace diagnoser supports only workspace schema version 4");
   }
 }
 
@@ -421,6 +427,7 @@ async function diagnoseValidatedWorkspace(
   adapters: DiagnosticAdapters,
   timeoutMs: number,
   semanticRuntime: SemanticRuntimeConfig,
+  skipDwh: boolean,
 ): Promise<ConnectorDiagnostics> {
   const evidenceField = descriptor.evidence?.source.type === "http"
     ? "evidence.source.authentication"
@@ -432,88 +439,97 @@ async function diagnoseValidatedWorkspace(
     variable,
   }));
   const diagnostics = [
-    ...[...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field)),
+    ...(skipDwh
+      ? []
+      : [...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field))),
     ...evidenceDiagnostics,
   ];
   if (diagnostics.length > 0) return { activatable: false, diagnostics };
 
-  const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs);
   let activatable = true;
-  const dwhValues = bindings.dwh.values;
-  const dwhField = (suffix: string) => bindingName(descriptor, suffix);
-  const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
-  let dwhRequest: ConnectorDiagnosticRequest | undefined;
-  if (bindings.dwh.transport === "rest_api") {
-    const diagnostic = descriptor.diagnostics?.dwh_rest;
-    const baseUrl = dwhValues[dwhField("BASE_URL")];
-    if (diagnostic && baseUrl) {
-      const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")];
-      if (diagnostic.auth === "none" || credentialFile !== undefined) {
+  if (!skipDwh) {
+    if (!descriptor.dwh) {
+      return { activatable: false, diagnostics: [diagnosticError("binding_missing", "dwh")] };
+    }
+    const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs);
+    const dwhValues = bindings.dwh.values;
+    const dwhField = (suffix: string) => bindingName(descriptor, suffix);
+    const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
+    let dwhRequest: ConnectorDiagnosticRequest | undefined;
+    if (bindings.dwh.transport === "rest_api") {
+      const diagnostic = descriptor.diagnostics?.dwh_rest;
+      const baseUrl = dwhValues[dwhField("BASE_URL")];
+      if (diagnostic && baseUrl) {
+        const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")];
+        if (diagnostic.auth === "none" || credentialFile !== undefined) {
+          dwhRequest = {
+            role: "dwh",
+            transport: "rest_api",
+            baseUrl,
+            credentialFile,
+            tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")],
+            resource: dwhResource,
+            timeoutMs: dwhTimeout,
+            signal: new AbortController().signal,
+            diagnostic,
+          };
+        }
+      }
+    } else if (bindings.dwh.transport === "postgres_direct") {
+      const host = dwhValues[dwhField("HOST")];
+      const port = numericBinding(dwhValues, dwhField("PORT"));
+      const user = dwhValues[dwhField("USER")];
+      const credentialFile = dwhValues[dwhField("PASSWORD_FILE")];
+      if (host && port && user && credentialFile) {
         dwhRequest = {
           role: "dwh",
-          transport: "rest_api",
-          baseUrl,
+          transport: "postgres_direct",
+          host,
+          port,
+          user,
           credentialFile,
           tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")],
           resource: dwhResource,
           timeoutMs: dwhTimeout,
           signal: new AbortController().signal,
-          diagnostic,
         };
       }
     }
-  } else if (bindings.dwh.transport === "postgres_direct") {
-    const host = dwhValues[dwhField("HOST")];
-    const port = numericBinding(dwhValues, dwhField("PORT"));
-    const user = dwhValues[dwhField("USER")];
-    const credentialFile = dwhValues[dwhField("PASSWORD_FILE")];
-    if (host && port && user && credentialFile) {
-      dwhRequest = {
-        role: "dwh",
-        transport: "postgres_direct",
-        host,
-        port,
-        user,
-        credentialFile,
-        tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")],
-        resource: dwhResource,
-        timeoutMs: dwhTimeout,
-        signal: new AbortController().signal,
-      };
+
+    if (!dwhRequest) {
+      diagnostics.push(diagnosticError("workspace_not_activatable"));
+      return { activatable: false, diagnostics };
     }
-  }
 
-  if (!dwhRequest) {
-    diagnostics.push(diagnosticError("workspace_not_activatable"));
-    return { activatable: false, diagnostics };
-  }
-
-  try {
-    const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({
-      ...dwhRequest,
-      signal,
-      timeoutMs: dwhTimeout,
-    }));
-    if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) {
+    try {
+      const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({
+        ...dwhRequest,
+        signal,
+        timeoutMs: dwhTimeout,
+      }));
+      if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) {
+        diagnostics.push(diagnosticError("connector_unavailable"));
+        activatable = false;
+      }
+    } catch {
       diagnostics.push(diagnosticError("connector_unavailable"));
       activatable = false;
     }
-  } catch {
-    diagnostics.push(diagnosticError("connector_unavailable"));
-    activatable = false;
   }
 
   try {
-    const vector = await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
-      baseUrl: semanticRuntime.internalQdrantUrl,
-      collection: descriptor.semantic_index.vector_store.collection,
-      timeoutMs,
-      signal,
-    }));
-    const expected = descriptor.semantic_index.vector_store;
-    if (vector.collection !== expected.collection
-      || vector.dimensions !== expected.dimensions
-      || vector.distance !== expected.distance) {
+    const expectedCollections = Object.values(workspaceVectorCollections(descriptor.workspace.id));
+    const vectors = await Promise.all(expectedCollections.map(async (collection) =>
+      await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
+        baseUrl: semanticRuntime.internalQdrantUrl,
+        collection,
+        timeoutMs,
+        signal,
+      }))));
+    if (vectors.some((vector, index) =>
+      vector.collection !== expectedCollections[index]
+      || vector.dimensions !== semanticRuntime.internalEmbeddingDimensions
+      || vector.distance !== "cosine")) {
       diagnostics.push(diagnosticError("semantic_index_incompatible"));
       activatable = false;
     }
@@ -529,10 +545,8 @@ async function diagnoseValidatedWorkspace(
       timeoutMs,
       signal,
     }));
-    if (semanticRuntime.internalEmbeddingModel !== descriptor.semantic_index.embedding.model
-      || semanticRuntime.internalEmbeddingDimensions !== descriptor.semantic_index.embedding.dimensions
-      || !embedding.available
-      || embedding.dimensions !== descriptor.semantic_index.embedding.dimensions) {
+    if (!embedding.available
+      || embedding.dimensions !== semanticRuntime.internalEmbeddingDimensions) {
       diagnostics.push(diagnosticError("semantic_index_incompatible"));
       activatable = false;
     }
@@ -569,11 +583,18 @@ export function createWorkspaceDiagnoser(
   return async function diagnose(
     workspace: WorkspaceDescriptor,
     bindings: RuntimeBindings,
-    _options: { writeProbe: boolean },
+    diagnosticOptions: WorkspaceDiagnosticOptions,
   ): Promise<ConnectorDiagnostics> {
     requireSupportedDescriptor(workspace);
     const descriptor = validateWorkspaceDescriptor(workspace);
-    return await diagnoseValidatedWorkspace(descriptor, bindings, adapters, timeoutMs, semanticRuntime);
+    return await diagnoseValidatedWorkspace(
+      descriptor,
+      bindings,
+      adapters,
+      timeoutMs,
+      semanticRuntime,
+      diagnosticOptions.skipDwh ?? false,
+    );
   };
 }
 

backend/src/workspaces/effective-config.ts

@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
 import { normalize } from "node:path";
 
 export interface CanonicalEffectiveConfig {
-  schemaVersion: 1;
+  schemaVersion: 3;
   dwh: CanonicalDwhConfig;
   vector: CanonicalVectorConfig;
   embedding: CanonicalEmbeddingConfig;
@@ -21,12 +21,16 @@ export interface CanonicalDwhConfig {
 }
 
 export interface CanonicalVectorConfig {
-  collection: string;
+  collections: {
+    reference: string;
+    memory: string;
+  };
   dimensions: number;
   distance: string;
 }
 
 export interface CanonicalEmbeddingConfig {
+  id: string;
   model: string;
   dimensions: number;
 }
@@ -119,7 +123,10 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
   if (!vector) {
     throw new TypeError("effective config is missing vector resources");
   }
-  const collection = requireString(vector, "collection");
+  const collections = asRecord(vector.collections);
+  if (!collections) {
+    throw new TypeError("effective config is missing vector collections");
+  }
   const semanticIndex = asRecord(rendered.semantic_index);
   const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined;
   const dimensions = vectorStore
@@ -128,7 +135,14 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
   const distance = vectorStore
     ? requireString(vectorStore, "distance")
     : (optionalString(vector, "distance") ?? "cosine");
-  return { collection, dimensions, distance };
+  return {
+    collections: {
+      reference: requireString(collections, "reference"),
+      memory: requireString(collections, "memory"),
+    },
+    dimensions,
+    distance,
+  };
 }
 
 function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbeddingConfig {
@@ -137,8 +151,10 @@ function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbed
   if (!embeddings) {
     throw new TypeError("effective config is missing embedding resources");
   }
+  const model = requireString(embeddings, "model");
   return {
-    model: requireString(embeddings, "model"),
+    id: optionalString(embeddings, "id") ?? `ollama/${model}`,
+    model,
     dimensions: requireNumber(embeddings, "dimensions"),
   };
 }
@@ -166,7 +182,7 @@ export function buildCanonicalEffectiveConfig(renderedConfig: unknown): Canonica
     throw new TypeError("effective config requires a rendered configuration object");
   }
   return {
-    schemaVersion: 1,
+    schemaVersion: 3,
     dwh: buildDwhConfig(rendered),
     vector: buildVectorConfig(rendered),
     embedding: buildEmbeddingConfig(rendered),
@@ -217,3 +233,20 @@ export function configFingerprint(renderedConfig: unknown): string {
 export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string {
   return sha256(effectiveConfigIdentity(workspaceId, renderedConfig));
 }
+
+/**
+ * Fingerprint every non-Catalog input consumed by complete preprocessing. The immutable Git
+ * revision covers the workspace descriptor and its revision-pinned Evidence tree; the effective
+ * configuration identity covers the Catalog-derived DWH binding and semantic runtime contract.
+ */
+export function preprocessingInputFingerprint(
+  workspaceId: string,
+  workspaceRevision: string,
+  renderedConfig: unknown,
+): string {
+  return sha256(JSON.stringify({
+    workspaceId,
+    workspaceRevision,
+    effectiveConfigIdentity: effectiveConfigIdentity(workspaceId, renderedConfig),
+  }));
+}

backend/src/workspaces/evidence/preprocessing.ts

@@ -22,6 +22,7 @@ export interface EvidencePreprocessingRequest {
   evidence: EvidenceConfig;
   job: EvidenceJobState;
   dryRun?: boolean;
+  consolidate?: boolean;
   httpPrivateHostAllowlist?: readonly string[];
 }
 
@@ -56,7 +57,7 @@ function isPrivateHost(hostname: string): boolean {
   return hostname.endsWith(".internal");
 }
 
-function evidencePolicy(
+export function evidencePolicy(
   evidence: EvidenceConfig,
   httpPrivateHostAllowlist?: readonly string[],
 ): EvidencePreprocessingOutcome | undefined {
@@ -95,13 +96,14 @@ function jobResult(job: EvidenceJobState): Pick<
   };
 }
 
-async function runEvidenceStage(
+export async function runEvidenceStage(
   request: EvidencePreprocessingRequest,
   deps: EvidencePreprocessingDependencies,
 ): Promise<EvidencePreprocessingOutcome> {
   const payload = await deps.runStage([
     "preprocess",
     "evidence",
+    ...(request.consolidate ? ["--consolidate"] : []),
     ...(request.dryRun ? ["--dry-run"] : []),
     ...(request.job.childRuns.evidence
       ? ["--resume", request.job.childRuns.evidence]
@@ -171,6 +173,14 @@ export async function continueEvidencePreprocessing(
   const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
   if (policy) return { ...policy, ...jobResult(request.job) };
   if (!request.job.completedStages.includes("evidence")) {
+    const preflight = await deps.evidencePreflight();
+    if (!preflight.ok) {
+      return {
+        status: "failed",
+        code: preflight.code,
+        ...jobResult(request.job),
+      };
+    }
     return await runEvidenceStage(request, deps);
   }
   return { status: "unchanged", code: "ok", ...jobResult(request.job) };

backend/src/workspaces/preprocessing-service.ts

@@ -1,22 +1,21 @@
-import { createHash, randomBytes } from "node:crypto";
-import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { renameSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import {
   continueEvidencePreprocessing,
-  preprocessEvidence as runEvidencePreprocessing,
+  evidencePolicy,
+  runEvidenceStage,
   type EvidencePreprocessingDependencies,
   type EvidencePreprocessingOutcome,
 } from "./evidence/preprocessing.js";
 import type { WorkspaceDescriptor } from "./schema.js";
 import {
   PreprocessingStateStore,
-  type FkReviewRecord,
   type PreprocessingJobState,
-  type SessionInventoryRow,
 } from "./preprocessing-state.js";
 import type { DeterministicRuntimeConfigLease } from "./runtime-config-lease.js";
-import { readAnnotationsSync } from "./annotations-sync.js";
-import { reconcileCollection } from "./qdrant-collection.js";
+import { buildCatalogMetadataSnapshot } from "../catalog/metadata-snapshot.js";
+import type { CatalogRepository } from "../catalog/types.js";
 
 export interface WorkspaceOperationResult {
   schemaVersion: 1;
@@ -27,7 +26,7 @@ export interface WorkspaceOperationResult {
     | "preprocessing_resume_mismatch" | "manual_review_required"
     | "evidence_materialization_required" | "effective_config_mismatch"
     | "semantic_index_incompatible" | "annotation_invalid"
-    | "egress_policy_refused";
+    | "egress_policy_refused" | "catalog_not_ready" | "preprocessing_clear_failed";
   workspaceId: string;
   workspaceRevision: string;
   descriptorBlob: string;
@@ -69,7 +68,6 @@ export interface WorkspacePreprocessingServiceDeps {
   dataRoot: string;
   acquireActiveRuntime(workspaceId: string): Promise<ActiveRuntime>;
   runChild(request: ChildProcessRequest): Promise<ChildProcessResult>;
-  listSessions(workspaceId: string): Promise<readonly SessionInventoryRow[]>;
   semanticPreflight(workspace: WorkspaceDescriptor): Promise<
     { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
   >;
@@ -77,6 +75,7 @@ export interface WorkspacePreprocessingServiceDeps {
     { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
   >;
   httpPrivateHostAllowlist?: readonly string[];
+  catalogRepository?: CatalogRepository;
 }
 
 interface RunScope {
@@ -85,10 +84,6 @@ interface RunScope {
   job: PreprocessingJobState;
 }
 
-function digest(value: string | Buffer): string {
-  return `sha256:${createHash("sha256").update(value).digest("hex")}`;
-}
-
 function baseResult(
   runtime: ActiveRuntime,
   operation: string,
@@ -115,41 +110,72 @@ function baseResult(
 export class WorkspacePreprocessingService {
   constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
 
-
-  async vectorInspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
+  async evidenceSources(options: { workspaceId: string; action: "refresh" | "decide";
+    sourceId?: string; revision?: string; decision?: "keep" | "replace"; actor?: string }): Promise<WorkspaceOperationResult> {
     const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-    const collection = runtime.workspace.semantic_index.vector_store.collection;
-    const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" });
-    if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] });
-    const body = await res.json() as any;
-    const info = body?.result;
-    const vectors = info?.config?.params?.vectors;
-    return baseResult(runtime, "vector inspect", "succeeded", "ok", {
-      counts: { dimensions: vectors?.size ?? 0 },
-      warnings: [`collection=${collection} distance=${vectors?.distance ?? "unknown"}`],
-    });
-  }
-
-  async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise<WorkspaceOperationResult> {
-    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-    const collection = runtime.workspace.semantic_index.vector_store.collection;
-    if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) {
-      return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] });
+    try {
+      if (options.action === "refresh") {
+        const refused = evidencePolicy(runtime.workspace.evidence, this.deps.httpPrivateHostAllowlist);
+        if (refused) return baseResult(runtime, "evidence refresh", "failed", refused.code);
+      }
+      const argv = ["evidence", "sources", options.action, "--json", "-c", "/dev/fd/3"];
+      if (options.action === "decide") {
+        if (!/^[a-f0-9]{64}$/.test(options.sourceId ?? "") || !/^[a-f0-9]{64}$/.test(options.revision ?? "")
+          || !["keep", "replace"].includes(options.decision ?? "")) throw new Error("Invalid source decision");
+        const preflight = await this.deps.evidencePreflight(runtime.workspace);
+        if (!preflight.ok) return baseResult(runtime, "evidence sources", "failed", preflight.code);
+        argv.push("--source-id", options.sourceId!, "--revision", options.revision!, "--decision", options.decision!);
+      }
+      argv.push("--actor", options.actor ?? "installation operator");
+      const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
+      const payload = JSON.parse(result.stdout);
+      if (result.exitCode !== 0 || payload.status !== "succeeded") throw new Error(
+        typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence source operation failed");
+      return baseResult(runtime, `evidence ${options.action}`, "succeeded", "ok", {
+        counts: this.numberRecord(payload.counts),
+        warnings: [options.action === "refresh" ? "Source comparisons are ready in Evidence management. Active content is unchanged."
+          : "Source decision activated locally. Commit and push the Evidence tree manually."],
+      });
+    } catch (error) {
+      return baseResult(runtime, `evidence ${options.action}`, "failed", "evidence_materialization_required", {
+        warnings: [error instanceof Error ? error.message : "Evidence source operation failed"],
+      });
     }
-    const q = `${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`;
-    const del = await fetch(q, { method: "DELETE" });
-    if (!del.ok && del.status !== 404) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection delete failed"] });
-    // Recreate the complete contract (dimensions + distance + the 8 required keyword indexes).
-    const recreated = await reconcileCollection({
-      baseUrl: runtime.configLease.semanticQdrantUrl,
-      collection,
-      dimensions: runtime.workspace.semantic_index.vector_store.dimensions,
-      distance: runtime.workspace.semantic_index.vector_store.distance,
-      mode: "self_heal",
-    });
-    if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] });
-    return baseResult(runtime, "vector rebuild", "succeeded", "ok", { warnings: [`recreated collection=${collection}`] });
   }
+
+  async consolidateEvidence(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
+    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
+    const preflight = await this.deps.evidencePreflight(runtime.workspace);
+    if (!preflight.ok) return baseResult(runtime, "evidence consolidate", "failed", preflight.code);
+    // Evidence has its own durable archive/corpus jobs. Do not alter Catalog readiness
+    // or the full preprocessing job when publishing this one component.
+    try {
+      const outcome = await runEvidenceStage({ evidence: runtime.workspace.evidence,
+        consolidate: true, job: { runId: randomBytes(16).toString("hex"), completedStages: [], childRuns: {} },
+      }, {
+        runStage: async argv => {
+          const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
+          const payload = JSON.parse(result.stdout);
+          if (result.exitCode !== 0 || payload.status !== "succeeded") {
+            return Promise.reject(new Error(typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence consolidation failed. Retry the command."));
+          }
+          return payload;
+        },
+        persistJob: () => undefined,
+        evidencePreflight: async () => preflight,
+        requireRunId: value => this.requireRunId(value),
+        numberRecord: value => this.numberRecord(value),
+      });
+      return baseResult(runtime, "evidence consolidate", "succeeded", "ok", {
+        ...outcome, warnings: ["Evidence is active locally. Catalog and Schema readiness are unchanged. Commit and push the Evidence files manually."],
+      });
+    } catch (error) {
+      return baseResult(runtime, "evidence consolidate", "failed", "evidence_materialization_required", {
+        warnings: [error instanceof Error ? error.message : "Evidence consolidation failed. Retry the command."],
+      });
+    }
+  }
+
   async inspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
     try {
       const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
@@ -174,232 +200,158 @@ export class WorkspacePreprocessingService {
     }
   }
 
-  async preprocessDwh(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
-    const scope = await this.startRun(options.workspaceId, "preprocess dwh", options.resumeRunId);
-    if (scope.job.completedStages.includes("dwh")) {
-      return baseResult(scope.runtime, "preprocess dwh", "unchanged", "ok", {
-        runId: scope.job.runId,
-        childRuns: scope.job.childRuns,
-        completedStages: [...scope.job.completedStages],
-      });
+  async run(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
+    if (!this.deps.catalogRepository) {
+      const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
+      return baseResult(runtime, "preprocess run", "failed", "catalog_not_ready");
     }
-    const payload = await this.runJsonStage(scope.runtime, [
-      "preprocess", "dwh", "--steps", "introspect,lsh",
-      ...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
-      "--json", "-c", "/dev/fd/3",
-    ]);
-    const childRun = this.requireRunId(payload.run_id);
-    scope.job.childRuns.dwh = childRun;
-    if (!scope.job.completedStages.includes("dwh")) scope.job.completedStages.push("dwh");
-    this.state(scope.runtime.workspaceId).writeJob(scope.job);
-    return baseResult(scope.runtime, "preprocess dwh", "succeeded", "ok", {
-      runId: scope.job.runId,
-      childRuns: { ...scope.job.childRuns },
-      completedStages: [...scope.job.completedStages],
-    });
+    return await this.runFromCatalog(options);
   }
 
-  async suggestFks(options: {
-    workspaceId: string;
-    fromSql?: ReadonlyArray<{ name: string; sql: string }>;
-    assume?: readonly string[];
-    resumeRunId?: string;
-  }): Promise<WorkspaceOperationResult> {
-    const scope = await this.startRun(options.workspaceId, "schema suggest-fks", options.resumeRunId);
-    return await this.runSuggestStage(scope, options.fromSql ?? [], options.assume ?? []);
-  }
-
-  async checkSchema(options: {
-    workspaceId: string;
-    annotationsYaml?: string;
-    reviewedCandidatesDigest?: string;
-  }): Promise<WorkspaceOperationResult> {
+  async clear(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
     const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-    const state = this.state(runtime.workspaceId);
-    if ((options.annotationsYaml === undefined) !== (options.reviewedCandidatesDigest === undefined)) {
-      return baseResult(runtime, "schema check", "failed", "annotation_invalid");
+    const repository = this.deps.catalogRepository;
+    if (!repository) return baseResult(runtime, "preprocess clear", "failed", "catalog_not_ready");
+    const cleared = await repository.clearPreprocessing(runtime.workspaceId);
+    if (cleared.kind !== "cleared") {
+      return baseResult(
+        runtime,
+        "preprocess clear",
+        "failed",
+        cleared.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
+      );
     }
-    if (options.reviewedCandidatesDigest === undefined) {
-      const payload = await this.runJsonStage(runtime, ["schema", "check", "--json", "-c", "/dev/fd/3"]);
-      return baseResult(runtime, "schema check", Number(payload.orphan_count ?? 0) === 0 ? "succeeded" : "failed", Number(payload.orphan_count ?? 0) === 0 ? "ok" : "annotation_invalid");
-    }
-    const reviewedCandidatesDigest = options.reviewedCandidatesDigest;
-    const runId = this.findRunIdByCandidateDigest(state, reviewedCandidatesDigest);
-    if (!runId) return baseResult(runtime, "schema check", "failed", "annotation_invalid");
-    const request = await this.withStagedInputs(runtime.workspaceId, [
-      { flag: "--annotations", name: "annotations.yaml", contents: options.annotationsYaml! },
-    ], async (argv) => await this.runJsonStage(runtime, [
-      "schema", "check", ...argv,
-      "--reviewed-candidates", reviewedCandidatesDigest,
-      "--json", "-c", "/dev/fd/3",
-    ]));
-    if (request.reviewed_candidates_digest !== reviewedCandidatesDigest || typeof request.annotations_digest !== "string") {
-      return baseResult(runtime, "schema check", "failed", "annotation_invalid", { runId });
-    }
-    // P5 supersedes the host-file FK review: schema check is read-only validation and never
-    // records a review. Only `schema accept` records a human review for the curated Git blob.
-    return baseResult(runtime, "schema check", "succeeded", "ok", { runId });
-  }
-
-  async acceptSchema(options: { workspaceId: string; runId: string; yes?: boolean }): Promise<WorkspaceOperationResult> {
-    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-    const state = this.state(runtime.workspaceId);
-    if (options.yes !== true) {
-      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
-        runId: options.runId,
-        warnings: ["accept requires --yes"],
-      });
-    }
-    if (!/^[0-9a-f]{32}$/.test(options.runId)) {
-      return baseResult(runtime, "schema accept", "failed", "annotation_invalid");
-    }
-    const candidate = state.readFkCandidates(options.runId);
-    if (candidate === undefined) {
-      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
-        runId: options.runId,
-        warnings: ["candidate run is unavailable"],
-      });
-    }
-    const synced = readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision);
-    if (synced === undefined || synced.contents.toString("utf8").trim() === "") {
-      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
-        runId: options.runId,
-        warnings: ["curated annotations are not synchronized"],
-      });
-    }
-    // The harness parser validates the curated blob against the physical schema; the recorded
-    // candidate digest must round-trip and the blob digest must match the synced destination.
-    const payload = await this.runJsonStage(runtime, [
-      "schema", "check", "--reviewed-candidates", candidate.digest, "--json", "-c", "/dev/fd/3",
-    ]);
-    if (payload.annotations_digest !== synced.contentDigest
-      || payload.reviewed_candidates_digest !== candidate.digest
-      || Number(payload.orphan_count ?? 0) !== 0) {
-      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { runId: options.runId });
-    }
-    const review = state.writeFkReview(options.runId, {
-      reviewedCandidatesDigest: candidate.digest,
-      annotationsDigest: synced.contentDigest,
-      workspaceRevision: runtime.workspaceRevision,
-      blobId: synced.blobId,
+    const result = await this.deps.runChild({
+      argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
+      configPath: runtime.configLease.path,
     });
-    const job = state.readJob(options.runId);
-    job.reviewDigest = review.digest;
-    if (!job.completedStages.includes("fk_review")) job.completedStages.push("fk_review");
-    state.writeJob(job);
-    return baseResult(runtime, "schema accept", "succeeded", "ok", {
-      runId: options.runId,
-      completedStages: [...job.completedStages],
-      artifactIdentities: [
-        { kind: "fk_review", digest: review.digest },
-        { kind: "annotations", digest: synced.contentDigest },
-      ],
+    if (result.exitCode !== 0) {
+      return baseResult(runtime, "preprocess clear", "failed", "preprocessing_clear_failed");
+    }
+    const payload = JSON.parse(result.stdout) as Record<string, unknown>;
+    return baseResult(runtime, "preprocess clear", "succeeded", "ok", {
+      counts: this.numberRecord(payload.counts),
     });
   }
 
-  async indexSchema(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
-    const scope = await this.startRun(options.workspaceId, "index-schema", options.resumeRunId);
-    const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
-    if (!semantic.ok) return baseResult(scope.runtime, "index-schema", "failed", semantic.code, { runId: scope.job.runId });
-    if (scope.job.completedStages.includes("schema_index")) {
-      return baseResult(scope.runtime, "index-schema", "unchanged", "ok", {
-        runId: scope.job.runId,
-        completedStages: [...scope.job.completedStages],
-      });
+  private async runFromCatalog(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
+    const scope = await this.startRun(options.workspaceId);
+    const repository = this.deps.catalogRepository!;
+    const fingerprint = scope.runtime.configLease.inputFingerprint;
+    const started = await repository.beginPreprocessing(scope.runtime.workspaceId, fingerprint);
+    if (started.kind !== "started") {
+      scope.job.status = "failed";
+      scope.state.writeJob(scope.job);
+      return baseResult(
+        scope.runtime,
+        "preprocess run",
+        "failed",
+        started.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
+        { warnings: [`catalog=${started.kind}`] },
+      );
     }
-    const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
-    const counts = this.numberRecord(payload.counts);
-    scope.job.completedStages.push("schema_index");
-    this.state(scope.runtime.workspaceId).writeJob(scope.job);
-    return baseResult(scope.runtime, "index-schema", "succeeded", "ok", {
-      runId: scope.job.runId,
-      completedStages: [...scope.job.completedStages],
-      counts,
-    });
-  }
 
-  async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
-    const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId);
-    const outcome = await runEvidencePreprocessing(
-      {
-        evidence: scope.runtime.workspace.evidence,
-        job: scope.job,
-        dryRun: options.dryRun,
-        httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
-      },
-      this.evidenceDependencies(scope),
-    );
-    return this.evidenceResult(scope, "preprocess evidence", outcome);
-  }
-
-  async run(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
-    const scope = await this.startRun(options.workspaceId, "preprocess run", options.resumeRunId);
-    if (!scope.job.completedStages.includes("dwh")) {
-      const payload = await this.runJsonStage(scope.runtime, [
-        "preprocess", "dwh", "--steps", "introspect,lsh",
-        ...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
-        "--json", "-c", "/dev/fd/3",
-      ]);
-      scope.job.childRuns.dwh = this.requireRunId(payload.run_id);
-      scope.job.completedStages.push("dwh");
-      this.state(scope.runtime.workspaceId).writeJob(scope.job);
-    }
-    if (!scope.job.completedStages.includes("fk_suggest")) {
-      const suggest = await this.runSuggestStage(scope, [], []);
-      if (suggest.code === "manual_review_required") return suggest;
-    }
-    const candidate = this.state(scope.runtime.workspaceId).readFkCandidates(scope.job.runId);
-    if (candidate && !scope.job.completedStages.includes("fk_review")) {
-      // P5: continuation requires a review accepted for this candidate whose accepted blob digest
-      // equals the current revision's synced annotations. A revision change (or a missing curated
-      // blob) therefore records a new review checkpoint instead of silently reusing the old one.
-      const accepted = this.findAcceptedReviewForDigest(scope.runtime.workspaceId, candidate.digest);
-      const currentDigest = this.currentAnnotationsDigest(scope.runtime);
-      if (accepted === undefined || currentDigest === undefined || accepted.annotationsDigest !== currentDigest) {
-        return baseResult(scope.runtime, "preprocess run", "blocked", "manual_review_required", {
-          runId: scope.job.runId,
-          childRuns: { ...scope.job.childRuns },
-          completedStages: [...scope.job.completedStages],
-          artifactIdentities: [{ kind: "fk_candidates", digest: candidate.digest }],
-        });
+    const revision = started.database.metadataContentRevision;
+    let finished = false;
+    let failureCode = "catalog_snapshot_failed";
+    try {
+      const snapshot = await buildCatalogMetadataSnapshot(
+        repository,
+        scope.runtime.workspaceId,
+        revision,
+      );
+      const snapshotPath = this.publishCatalogSnapshot(
+        scope.runtime.workspaceId,
+        JSON.stringify(snapshot),
+      );
+      this.completeStages(scope, "catalog_snapshot");
+      const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
+      if (!semantic.ok) {
+        await repository.finishPreprocessing(
+          scope.runtime.workspaceId,
+          revision,
+          fingerprint,
+          { status: "failed", errorCode: semantic.code },
+        );
+        scope.job.status = "failed";
+        scope.state.writeJob(scope.job);
+        finished = true;
+        return baseResult(scope.runtime, "preprocess run", "failed", semantic.code);
       }
-      scope.job.reviewDigest = accepted.reviewedCandidatesDigest;
-      scope.job.completedStages.push("fk_review");
-      this.state(scope.runtime.workspaceId).writeJob(scope.job);
+
+      failureCode = "schema_index_failed";
+      const payload = await this.runJsonStage(scope.runtime, [
+        "preprocess",
+        "catalog",
+        "--catalog-metadata",
+        snapshotPath,
+        "--json",
+        "-c",
+        "/dev/fd/3",
+      ]);
+      this.completeStages(scope, "catalog_metadata", "lsh", "schema_index");
+      failureCode = "evidence_preprocessing_failed";
+      const outcome = await continueEvidencePreprocessing(
+        {
+          evidence: scope.runtime.workspace.evidence,
+          job: scope.job,
+          httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
+          priorCounts: this.numberRecord(payload.counts),
+        },
+        this.evidenceDependencies(scope),
+      );
+      if (!["succeeded", "unchanged"].includes(outcome.status)) {
+        await repository.finishPreprocessing(
+          scope.runtime.workspaceId,
+          revision,
+          fingerprint,
+          { status: "failed", errorCode: outcome.code },
+        );
+        scope.job.status = "failed";
+        scope.state.writeJob(scope.job);
+        finished = true;
+        return this.evidenceResult(scope, outcome);
+      }
+      // Evidence has been published. The only remaining operation is the atomic Catalog commit.
+      this.completeStages(scope, "evidence");
+      const completed = await repository.finishPreprocessing(
+        scope.runtime.workspaceId,
+        revision,
+        fingerprint,
+        { status: "succeeded" },
+      );
+      if (!completed) throw new Error("catalog preprocessing completion lost its lease");
+      scope.job.status = "succeeded";
+      scope.state.writeJob(scope.job);
+      finished = true;
+      return this.evidenceResult(scope, outcome);
+    } catch (error) {
+      if (!finished) {
+        await repository.finishPreprocessing(
+          scope.runtime.workspaceId,
+          revision,
+          fingerprint,
+          { status: "failed", errorCode: failureCode },
+        );
+      }
+      scope.job.status = "failed";
+      scope.state.writeJob(scope.job);
+      throw error;
     }
-    const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
-    if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId });
-    let schemaCounts: Record<string, number> | undefined;
-    if (!scope.job.completedStages.includes("schema_index")) {
-      const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
-      scope.job.completedStages.push("schema_index");
-      this.state(scope.runtime.workspaceId).writeJob(scope.job);
-      schemaCounts = this.numberRecord(payload.counts);
-    }
-    const outcome = await continueEvidencePreprocessing(
-      {
-        evidence: scope.runtime.workspace.evidence,
-        job: scope.job,
-        httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
-        priorCounts: schemaCounts,
-      },
-      this.evidenceDependencies(scope),
-    );
-    return this.evidenceResult(scope, "preprocess run", outcome);
   }
 
-  private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise<RunScope> {
+  private async startRun(workspaceId: string): Promise<RunScope> {
     const runtime = await this.deps.acquireActiveRuntime(workspaceId);
     const state = this.state(runtime.workspaceId);
-    await state.assertSessionInventoryCompatible(runtime.workspaceRevision, await this.deps.listSessions(runtime.workspaceId));
     const job = await state.beginJob({
-      operation,
-      runId: resumeRunId,
+      operation: "preprocess run",
       workspaceRevision: runtime.workspaceRevision,
       descriptorBlob: runtime.descriptorBlob,
       catalogBlob: runtime.catalogBlob,
       configDigest: runtime.configLease.configDigest,
       bindingDigest: runtime.configLease.bindingDigest,
+      embeddingId: runtime.configLease.effectiveConfig.embedding.id,
+      embeddingDimensions: runtime.configLease.effectiveConfig.embedding.dimensions,
     });
     return { runtime, state, job };
   }
@@ -408,6 +360,28 @@ export class WorkspacePreprocessingService {
     return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId });
   }
 
+  private completeStages(scope: RunScope, ...stages: string[]): void {
+    for (const stage of stages) {
+      if (!scope.job.completedStages.includes(stage)) scope.job.completedStages.push(stage);
+    }
+    scope.state.writeJob(scope.job);
+  }
+
+  private publishCatalogSnapshot(workspaceId: string, contents: string): string {
+    const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing");
+    mkdirSync(root, { recursive: true, mode: 0o700 });
+    const target = join(root, "catalog-metadata.json");
+    const staging = join(root, `.catalog-metadata-${randomBytes(6).toString("hex")}.json`);
+    try {
+      writeFileSync(staging, contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
+      renameSync(staging, target);
+      return target;
+    } catch (error) {
+      rmSync(staging, { force: true });
+      throw error;
+    }
+  }
+
   private evidenceDependencies(scope: RunScope): EvidencePreprocessingDependencies {
     return {
       runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
@@ -420,50 +394,10 @@ export class WorkspacePreprocessingService {
 
   private evidenceResult(
     scope: RunScope,
-    operation: "preprocess evidence" | "preprocess run",
     outcome: EvidencePreprocessingOutcome,
   ): WorkspaceOperationResult {
     const { status, code, ...extra } = outcome;
-    return baseResult(scope.runtime, operation, status, code, extra);
-  }
-
-  private async runSuggestStage(
-    scope: RunScope,
-    fromSql: ReadonlyArray<{ name: string; sql: string }>,
-    assume: readonly string[],
-  ): Promise<WorkspaceOperationResult> {
-    const payload = await this.withStagedInputs(scope.runtime.workspaceId, fromSql.map((entry) => ({
-      flag: "--from-sql",
-      name: entry.name,
-      contents: entry.sql,
-    })), async (stagedArgv) => await this.runJsonStage(scope.runtime, [
-      "schema", "suggest-fks", ...stagedArgv,
-      ...assume.flatMap((value) => ["--assume", value]),
-      "--json", "-c", "/dev/fd/3",
-    ]));
-    const candidateCount = Number(payload.candidate_count ?? 0);
-    const candidateYaml = typeof payload.candidate_yaml === "string" ? payload.candidate_yaml : "";
-    let artifactIdentities: Array<{ kind: string; digest: string }> | undefined;
-    if (candidateCount > 0) {
-      const persisted = this.state(scope.runtime.workspaceId).writeFkCandidates(scope.job.runId, candidateYaml);
-      scope.job.candidateDigest = persisted.digest;
-      artifactIdentities = [{ kind: "fk_candidates", digest: persisted.digest }];
-    }
-    if (!scope.job.completedStages.includes("fk_suggest")) scope.job.completedStages.push("fk_suggest");
-    this.state(scope.runtime.workspaceId).writeJob(scope.job);
-    const resultExtra = {
-      runId: scope.job.runId,
-      completedStages: [...scope.job.completedStages],
-      ...(artifactIdentities ? { artifactIdentities } : {}),
-      ...(candidateYaml.length > 0 ? { suggestedFksYaml: candidateYaml } : {}),
-    };
-    if (candidateCount > 0) {
-      return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "blocked", "manual_review_required", {
-        ...resultExtra,
-        childRuns: { ...scope.job.childRuns },
-      });
-    }
-    return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "succeeded", "ok", resultExtra);
+    return baseResult(scope.runtime, "preprocess run", status, code, extra);
   }
 
   private async runJsonStage(runtime: ActiveRuntime, argv: string[]): Promise<Record<string, unknown>> {
@@ -482,54 +416,5 @@ export class WorkspacePreprocessingService {
     return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([key, nested]) => [key, Number(nested)]));
   }
 
-  private async withStagedInputs<T>(
-    workspaceId: string,
-    inputs: ReadonlyArray<{ flag: string; name: string; contents: string }>,
-    fn: (argv: string[]) => Promise<T>,
-  ): Promise<T> {
-    if (inputs.length === 0) return await fn([]);
-    const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing", `.stage-${randomBytes(6).toString("hex")}`);
-    mkdirSync(root, { recursive: true, mode: 0o700 });
-    const argv: string[] = [];
-    const paths: string[] = [];
-    try {
-      for (const input of inputs) {
-        const path = join(root, input.name);
-        writeFileSync(path, input.contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
-        paths.push(path);
-        argv.push(input.flag, path);
-      }
-      return await fn(argv);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  }
-
-  private currentAnnotationsDigest(runtime: ActiveRuntime): string | undefined {
-    return readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision)?.contentDigest;
-  }
-
-  private findAcceptedReviewForDigest(
-    workspaceId: string,
-    digestValue: string,
-  ): FkReviewRecord | undefined {
-    const state = this.state(workspaceId);
-    for (const entry of readdirSync(state.fkReviewsDirectory(), { withFileTypes: true })) {
-      if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.json$/.test(entry.name)) continue;
-      const runId = entry.name.slice(0, -".json".length);
-      const review = state.readFkReview(runId);
-      if (review && review.reviewedCandidatesDigest === digestValue) return review;
-    }
-    return undefined;
-  }
-
-  private findRunIdByCandidateDigest(state: PreprocessingStateStore, digestValue: string): string | undefined {
-    for (const entry of readdirSync(state.fkCandidatesDirectory(), { withFileTypes: true })) {
-      if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.yaml$/.test(entry.name)) continue;
-      const runId = entry.name.slice(0, -".yaml".length);
-      if (state.readFkCandidates(runId)?.digest === digestValue) return runId;
-    }
-    return undefined;
-  }
 
 }

backend/src/workspaces/preprocessing-state.ts

@@ -44,11 +44,13 @@ export interface BeginPreprocessingJobOptions {
   catalogBlob: string;
   configDigest: string;
   bindingDigest: string;
+  embeddingId: string;
+  embeddingDimensions: number;
   runId?: string;
 }
 
 export interface PreprocessingJobState {
-  schemaVersion: 1;
+  schemaVersion: 2;
   runId: string;
   operation: string;
   workspaceId: string;
@@ -57,6 +59,8 @@ export interface PreprocessingJobState {
   catalogBlob: string;
   configDigest: string;
   bindingDigest: string;
+  embeddingId: string;
+  embeddingDimensions: number;
   completedStages: string[];
   childRuns: Record<string, string>;
   status: "active" | "succeeded" | "blocked" | "failed";
@@ -146,7 +150,7 @@ function decodeJob(value: unknown): PreprocessingJobState {
   }
   const record = value as Record<string, unknown>;
   if (
-    record.schemaVersion !== 1
+    record.schemaVersion !== 2
     || typeof record.runId !== "string"
     || typeof record.operation !== "string"
     || typeof record.workspaceId !== "string"
@@ -155,6 +159,8 @@ function decodeJob(value: unknown): PreprocessingJobState {
     || typeof record.catalogBlob !== "string"
     || typeof record.configDigest !== "string"
     || typeof record.bindingDigest !== "string"
+    || typeof record.embeddingId !== "string"
+    || typeof record.embeddingDimensions !== "number"
     || !Array.isArray(record.completedStages)
     || typeof record.childRuns !== "object" || record.childRuns === null || Array.isArray(record.childRuns)
     || !["active", "succeeded", "blocked", "failed"].includes(String(record.status))
@@ -192,6 +198,7 @@ export class PreprocessingStateStore {
   runtimeConfigDirectory(): string { return join(this.root, "runtime-config"); }
   runtimeConfigManifestDirectory(): string { return join(this.root, "runtime-config-manifests"); }
   jobsDirectory(): string { return join(this.root, "jobs"); }
+  latestJobPath(): string { return join(this.root, "latest-job.json"); }
   fkCandidatesDirectory(): string { return join(this.root, "fk-candidates"); }
   fkReviewsDirectory(): string { return join(this.root, "fk-reviews"); }
   jobPath(runId: string): string { return join(this.jobsDirectory(), `${validateRunId(runId)}.json`); }
@@ -275,13 +282,15 @@ export class PreprocessingStateStore {
         || existing.catalogBlob !== options.catalogBlob
         || existing.configDigest !== options.configDigest
         || existing.bindingDigest !== options.bindingDigest
+        || existing.embeddingId !== options.embeddingId
+        || existing.embeddingDimensions !== options.embeddingDimensions
       ) {
         throw new PreprocessingStateError(
           "preprocessing_resume_mismatch",
           "Workspace preprocessing resume no longer matches the pinned revision",
         );
       }
-      return existing;
+      return this.writeJob(existing);
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
         if (error instanceof PreprocessingStateError) throw error;
@@ -295,7 +304,7 @@ export class PreprocessingStateStore {
       }
     }
     const job: PreprocessingJobState = {
-      schemaVersion: 1,
+      schemaVersion: 2,
       runId,
       operation: options.operation,
       workspaceId: this.options.workspaceId,
@@ -304,23 +313,36 @@ export class PreprocessingStateStore {
       catalogBlob: options.catalogBlob,
       configDigest: options.configDigest,
       bindingDigest: options.bindingDigest,
+      embeddingId: options.embeddingId,
+      embeddingDimensions: options.embeddingDimensions,
       completedStages: [],
       childRuns: {},
       status: "active",
     };
-    writeAtomicFile(path, `${JSON.stringify(job)}
-`, 0o600);
-    return job;
+    return this.writeJob(job);
   }
 
   readJob(runId: string): PreprocessingJobState {
     return decodeJob(JSON.parse(readTrustedFile(this.jobPath(runId))));
   }
 
+  readLatestJob(): PreprocessingJobState | undefined {
+    try {
+      return decodeJob(JSON.parse(readTrustedFile(this.latestJobPath())));
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+      throw error;
+    }
+  }
+
   writeJob(job: PreprocessingJobState): PreprocessingJobState {
     this.ensureLayout();
-    writeAtomicFile(this.jobPath(job.runId), `${JSON.stringify(job)}
-`, 0o600);
+    const contents = `${JSON.stringify(job)}
+`;
+    writeAtomicFile(this.jobPath(job.runId), contents, 0o600);
+    // This fixed pointer is the sole status surface for operators. Per-run files remain an
+    // internal resume mechanism and are never exposed as history.
+    writeAtomicFile(this.latestJobPath(), contents, 0o600);
     return job;
   }
 

backend/src/workspaces/registry.ts

@@ -560,7 +560,7 @@ export class WorkspaceRegistry {
             });
           }
         }
-        const collection = workspace.semantic_index.vector_store.collection;
+        const collection = workspace.workspace.id;
         const owner = collectionOwners.get(collection);
         if (owner !== undefined) {
           throw new Error(`duplicate qdrant collection ownership: ${collection} (${owner}, ${id})`);

backend/src/workspaces/runtime-config-lease.ts

@@ -23,7 +23,7 @@ import {
   canonicalEffectiveConfigJson,
   configFingerprint,
   effectiveConfigIdentity,
-  inputFingerprint,
+  preprocessingInputFingerprint,
   type CanonicalEffectiveConfig,
 } from "./effective-config.js";
 import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
@@ -41,6 +41,8 @@ import {
 } from "./runtime-renderer.js";
 import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js";
 import type { WorkspaceRegistryConfig } from "./types.js";
+import { resolveCatalogRuntimeBinding } from "../catalog/runtime-binding.js";
+import type { WorkspaceDatabase } from "../catalog/types.js";
 
 export interface RuntimeConfigLease {
   path: string;
@@ -246,8 +248,6 @@ function readSnapshotWorkspace(snapshotPath: string): {
 function runtimePaths(
   dataRoot: string,
   workspaceId: string,
-  workspaceRevision?: string,
-  effectiveRelationshipsPath?: string,
 ): RuntimePaths {
   if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
   const root = join(dataRoot, "sessions", workspaceId);
@@ -256,12 +256,7 @@ function runtimePaths(
     artifacts: join(root, "artifacts"),
     indexes: join(root, "indexes"),
     memory: join(root, "memory"),
-    ...(workspaceRevision === undefined
-      ? {}
-      : { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }),
-    ...(effectiveRelationshipsPath === undefined
-      ? {}
-      : { effective_relationships: effectiveRelationshipsPath }),
+    catalog_metadata_snapshot: join(root, "preprocessing", "catalog-metadata.json"),
   };
 }
 
@@ -309,19 +304,32 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
   dataRoot: string;
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
-  effectiveRelationshipsPath?: string;
   workspaceSecretStore?: WorkspaceSecretStore;
+  catalogDatabase?: WorkspaceDatabase;
 }): RenderedWorkspaceRuntime {
-  const secretLease = options.workspaceSecretStore === undefined
+  if (options.catalogDatabase && !options.workspaceSecretStore) {
+    throw new Error("Catalog runtime binding requires the workspace secret store");
+  }
+  const catalogLease = options.catalogDatabase === undefined
+    ? undefined
+    : resolveCatalogRuntimeBinding({
+      workspace: options.workspace,
+      database: options.catalogDatabase,
+      environment: process.env,
+      secretRoots: options.secretRoots,
+      secretStore: options.workspaceSecretStore!,
+    });
+  const runtimeWorkspace = catalogLease?.workspace ?? options.workspace;
+  const secretLease = catalogLease !== undefined || options.workspaceSecretStore === undefined
     ? undefined
     : resolveRuntimeBindingsWithWorkspaceSecrets(
-      options.workspace,
+      runtimeWorkspace,
       process.env,
       options.secretRoots,
       options.workspaceSecretStore,
     );
-  const bindings = secretLease?.bindings
-    ?? resolveRuntimeBindings(options.workspace, process.env, options.secretRoots);
+  const bindings = catalogLease?.bindings ?? secretLease?.bindings
+    ?? resolveRuntimeBindings(runtimeWorkspace, process.env, options.secretRoots);
   const overlay = installationOverlay(options.harnessDir, options.configPath);
   const context: RuntimeRenderContext = {
     workspaceId: options.workspaceId,
@@ -334,32 +342,26 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
       workspaceId: options.workspaceId,
       workspaceRevision: options.workspaceRevision,
       revisionContentRoot: options.revisionContentRoot,
-      runtimePaths: runtimePaths(
-        options.dataRoot,
-        options.workspaceId,
-        options.workspaceRevision,
-        options.effectiveRelationshipsPath,
-      ),
+      runtimePaths: runtimePaths(options.dataRoot, options.workspaceId),
       installationOverlay: overlay,
       bindings,
       bindingDigest: stableBindingDigest(bindings),
       semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
-      releaseSecrets: () => secretLease?.release(),
+      releaseSecrets: () => {
+        catalogLease?.release();
+        secretLease?.release();
+      },
       renderedConfig: renderRuntimeConfig(
-        options.workspace,
+        runtimeWorkspace,
         bindings,
-        runtimePaths(
-          options.dataRoot,
-          options.workspaceId,
-          options.workspaceRevision,
-          options.effectiveRelationshipsPath,
-        ),
+        runtimePaths(options.dataRoot, options.workspaceId),
         context,
         overlay,
         options.semanticRuntime,
       ),
     };
   } catch (error) {
+    catalogLease?.release();
     secretLease?.release();
     throw error;
   }
@@ -372,8 +374,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
   dataRoot: string;
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
-  effectiveRelationshipsPath?: string;
   workspaceSecretStore?: WorkspaceSecretStore;
+  catalogDatabase?: WorkspaceDatabase;
 }): RenderedWorkspaceRuntime {
   const snapshot = readSnapshotWorkspace(options.snapshotPath);
   return renderWorkspaceRuntimeFromWorkspace({
@@ -386,8 +388,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
     dataRoot: options.dataRoot,
     secretRoots: options.secretRoots,
     semanticRuntime: options.semanticRuntime,
-    effectiveRelationshipsPath: options.effectiveRelationshipsPath,
     workspaceSecretStore: options.workspaceSecretStore,
+    catalogDatabase: options.catalogDatabase,
   });
 }
 
@@ -401,6 +403,7 @@ export async function renderActiveWorkspaceRuntime(options: {
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
   workspaceSecretStore?: WorkspaceSecretStore;
+  catalogDatabase?: WorkspaceDatabase;
 }): Promise<ActiveRenderedWorkspaceRuntime> {
   // The persisted active state may reference host-side snapshot paths (written by another
   // process or installation). Read the active state directly and resolve the immutable snapshot
@@ -431,6 +434,7 @@ export async function renderActiveWorkspaceRuntime(options: {
     secretRoots: options.secretRoots,
     semanticRuntime: options.semanticRuntime,
     workspaceSecretStore: options.workspaceSecretStore,
+    catalogDatabase: options.catalogDatabase,
   });
   return {
     ...rendered,
@@ -480,6 +484,7 @@ export async function publishDeterministicRuntimeConfigLease(options: {
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
   workspaceSecretStore?: WorkspaceSecretStore;
+  catalogDatabase?: WorkspaceDatabase;
 }): Promise<DeterministicRuntimeConfigLease> {
   const rendered = await renderActiveWorkspaceRuntime(options);
   const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
@@ -487,8 +492,15 @@ export async function publishDeterministicRuntimeConfigLease(options: {
   const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
   const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
   const configFingerprintValue = configFingerprint(renderedConfigObject);
-  const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject);
-  const identitySuffix = inputFingerprintValue.slice(7, 23);
+  const inputFingerprintValue = preprocessingInputFingerprint(
+    rendered.workspaceId,
+    rendered.workspaceRevision,
+    renderedConfigObject,
+  );
+  // The Catalog input identity intentionally excludes representation-only changes.
+  // A lease also identifies its bytes, so a new renderer never collides with an
+  // immutable config produced by an earlier release for the same Catalog inputs.
+  const identitySuffix = sha256(inputFingerprintValue + "\n" + publishedConfig).slice(7, 23);
 
   const preprocessingRoot = ensureTrustedDirectory(join(
     options.dataRoot,

backend/src/workspaces/runtime-renderer.ts

@@ -1,8 +1,9 @@
-import { basename, join } from "node:path";
+import { basename, dirname, join } from "node:path";
 import { stringify } from "yaml";
 import { buildInstallationContract } from "./contracts.js";
 import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js";
 import type { ResolvedBinding, ResolvedEvidenceBinding, RuntimeBindings } from "./bindings.js";
+import { workspaceVectorCollections } from "./vector-collections.js";
 export type { RuntimeBindings } from "./bindings.js";
 
 export interface RuntimePaths {
@@ -10,10 +11,8 @@ export interface RuntimePaths {
   artifacts: string;
   indexes: string;
   memory: string;
-  /** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */
-  annotations_root?: string;
-  /** Immutable Catalog projection used as the exclusive runtime relationship source. */
-  effective_relationships?: string;
+  /** Current backend-produced projection of the PostgreSQL Metadata Catalog. */
+  catalog_metadata_snapshot?: string;
 }
 
 export interface RuntimeIdentity {
@@ -34,6 +33,7 @@ export interface RuntimeInstallationOverlay {
 export interface SemanticRuntimeConfig {
   internalQdrantUrl: string;
   internalEmbeddingUrl: string;
+  internalEmbeddingId?: string;
   internalEmbeddingModel: string;
   internalEmbeddingDimensions: number;
 }
@@ -41,6 +41,7 @@ export interface SemanticRuntimeConfig {
 export const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
   internalQdrantUrl: "http://qdrant:6333",
   internalEmbeddingUrl: "http://embedding:11434",
+  internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
   internalEmbeddingModel: "qwen3-embedding:0.6b",
   internalEmbeddingDimensions: 1024,
 };
@@ -178,6 +179,7 @@ function renderEvidence(
   return {
     evidence: {
       ...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}),
+      local_archive_root: join(dirname(dirname(context.revisionContentRoot)), "repo", context.workspaceId),
       sources: [renderedSource],
     },
     vector: {
@@ -202,16 +204,16 @@ function placeholderConnection(identity: { database: string; schema: string }):
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Runtime renderer supports only workspace schema version 3");
+    throw new Error("Runtime renderer supports only workspace schema version 4");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Runtime renderer supports only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Runtime renderer supports only workspace schema version 4");
   }
 }
 
-/** Render the schema-v3 compatibility fields consumed by the current Python harness. */
+/** Render the schema-v4 compatibility fields consumed by the current Python harness. */
 export function renderRuntimeConfig(
   workspace: WorkspaceDescriptor,
   bindings: RuntimeBindings,
@@ -222,6 +224,7 @@ export function renderRuntimeConfig(
 ): string {
   requireSupportedDescriptor(workspace);
   const descriptor = validateWorkspaceDescriptor(workspace);
+  if (!descriptor.dwh) throw new Error("runtime configuration requires a Catalog database binding");
   const contract = buildInstallationContract(descriptor);
   const name = (role: "DWH" | "EVIDENCE", suffix: string) => {
     const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix);
@@ -241,6 +244,7 @@ export function renderRuntimeConfig(
   }
 
   const dwhIdentity = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
+  const vectorCollections = workspaceVectorCollections(descriptor.workspace.id);
   const database = bindings.dwh.transport === "postgres_direct"
     ? { ...directConnection(bindings.dwh, {
       host: name("DWH", "HOST"),
@@ -263,16 +267,32 @@ export function renderRuntimeConfig(
     ...(installation.profile === undefined ? {} : { profile: installation.profile }),
     language: descriptor.workspace.language,
     database,
-    semantic_index: descriptor.semantic_index,
+    semantic_index: {
+      vector_store: {
+        engine: "qdrant",
+        collections: vectorCollections,
+        dimensions: semanticRuntime.internalEmbeddingDimensions,
+        distance: "cosine",
+      },
+      embedding: {
+        provider: "ollama_internal",
+        id: semanticRuntime.internalEmbeddingId
+          ?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
+        model: semanticRuntime.internalEmbeddingModel,
+        dimensions: semanticRuntime.internalEmbeddingDimensions,
+      },
+    },
     resources: {
       vector: {
         engine: "qdrant",
         base_url: semanticRuntime.internalQdrantUrl,
-        collection: descriptor.semantic_index.vector_store.collection,
+        collections: vectorCollections,
       },
       embeddings: {
         provider: "ollama_internal",
         base_url: semanticRuntime.internalEmbeddingUrl,
+        id: semanticRuntime.internalEmbeddingId
+          ?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
         model: semanticRuntime.internalEmbeddingModel,
         dimensions: semanticRuntime.internalEmbeddingDimensions,
       },

backend/src/workspaces/schema.ts

@@ -35,7 +35,7 @@ export interface CanonicalDiagnostics {
 }
 
 interface WorkspaceMetadata {
-  schema_version: 3;
+  schema_version: 4;
   id: string;
   name: string;
   description?: string;
@@ -51,31 +51,13 @@ interface WorkspaceDwh {
   supported_transports: DwhTransport[];
 }
 
-interface WorkspaceBase<TVectorStore> {
+interface WorkspaceBase {
   workspace: WorkspaceMetadata;
-  dwh: WorkspaceDwh;
-  semantic_index: {
-    vector_store: TVectorStore;
-    embedding: {
-      provider: "ollama_internal";
-      model: "qwen3-embedding:0.6b";
-      dimensions: 1024;
-    };
-  };
-  llm_policy: {
-    default?: `${string}/${string}`;
-    allowed: `${string}/${string}`[];
-  };
+  /** Legacy connection block; current descriptors bind their database through PostgreSQL. */
+  dwh?: WorkspaceDwh;
   diagnostics?: Pick<CanonicalDiagnostics, "dwh_rest">;
 }
 
-interface QdrantVectorStore {
-  engine: "qdrant";
-  collection: string;
-  dimensions: 1024;
-  distance: "cosine";
-}
-
 export interface EvidencePolicy {
   max_chunk_chars: number;
   retain_published_generations: number;
@@ -120,12 +102,12 @@ export interface WorkspaceEvidence {
   policy: EvidencePolicy;
 }
 
-export interface WorkspaceV3 extends WorkspaceBase<QdrantVectorStore> {
+export interface WorkspaceV4 extends WorkspaceBase {
   evidence?: WorkspaceEvidence;
 }
 
-export type CanonicalWorkspace = WorkspaceV3;
-export type WorkspaceDescriptor = WorkspaceV3;
+export type CanonicalWorkspace = WorkspaceV4;
+export type WorkspaceDescriptor = WorkspaceV4;
 
 const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, {
   message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$",
@@ -135,9 +117,6 @@ const identifier = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, {
 });
 const port = z.number().int().min(1).max(65_535);
 const timeoutMs = z.number().int().positive();
-const modelReference = z.string().regex(/^[^/\s]+\/[^/\s]+$/, {
-  message: "model must use provider/model syntax",
-});
 
 function isOriginRelativeDiagnosticPath(value: string): boolean {
   return /^\/(?!\/)[^\\\u0000-\u001F\u007F?#]*$/.test(value) && !/%5c/i.test(value);
@@ -166,21 +145,6 @@ const dwhSchema = z.object({
   timeout_ms: timeoutMs.optional(),
   supported_transports: z.array(z.enum(DWH_TRANSPORTS)).min(1),
 }).strict();
-const internalEmbeddingSchema = z.object({
-  provider: z.literal("ollama_internal"),
-  model: z.literal("qwen3-embedding:0.6b"),
-  dimensions: z.literal(1024),
-}).strict();
-const qdrantVectorStoreSchema = z.object({
-  engine: z.literal("qdrant"),
-  collection: workspaceId,
-  dimensions: z.literal(1024),
-  distance: z.literal("cosine"),
-}).strict();
-const llmPolicySchema = z.object({
-  default: modelReference.optional(),
-  allowed: z.array(modelReference).min(1),
-}).strict();
 
 const positiveSafeInteger = z.number().int().safe().positive();
 const nonnegativeSafeInteger = z.number().int().safe().nonnegative();
@@ -365,8 +329,9 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert
 }
 
 function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
-  unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
-  unique(workspace.llm_policy.allowed, context, ["llm_policy", "allowed"]);
+  if (workspace.dwh) {
+    unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
+  }
 
   if (workspace.evidence?.source.type === "filesystem") {
     const expected = `${workspace.workspace.id}/evidence`;
@@ -379,21 +344,8 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
     }
   }
 
-  if (workspace.semantic_index.vector_store.dimensions !== workspace.semantic_index.embedding.dimensions) {
-    context.addIssue({
-      code: "custom",
-      path: ["semantic_index", "embedding", "dimensions"],
-      message: "embedding dimensions must match vector store dimensions",
-    });
-  }
-  if (workspace.llm_policy.default && !workspace.llm_policy.allowed.includes(workspace.llm_policy.default)) {
-    context.addIssue({
-      code: "custom",
-      path: ["llm_policy", "default"],
-      message: "LLM default must be included in the allowlist",
-    });
-  }
-  if (workspace.diagnostics?.dwh_rest && !workspace.dwh.supported_transports.includes("rest_api")) {
+  if (workspace.diagnostics?.dwh_rest
+    && !workspace.dwh?.supported_transports.includes("rest_api")) {
     context.addIssue({
       code: "custom",
       path: ["diagnostics", "dwh_rest"],
@@ -402,25 +354,20 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
   }
 }
 
-const WorkspaceV3Schema = z.object({
-  dwh: dwhSchema,
-  llm_policy: llmPolicySchema,
+const WorkspaceV4Schema = z.object({
+  dwh: dwhSchema.optional(),
   evidence: workspaceEvidenceSchema.optional(),
   diagnostics: z.object({
     dwh_rest: dwhRestDiagnostic.optional(),
   }).strict().optional(),
   workspace: z.object({
-    schema_version: z.literal(3), id: workspaceId, name: z.string().trim().min(1),
+    schema_version: z.literal(4), id: workspaceId, name: z.string().trim().min(1),
     description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]),
   }).strict(),
-  semantic_index: z.object({
-    vector_store: qdrantVectorStoreSchema,
-    embedding: internalEmbeddingSchema,
-  }).strict(),
 }).strict().superRefine(workspaceInvariants);
-const WorkspaceDescriptorSchema = WorkspaceV3Schema;
+const WorkspaceDescriptorSchema = WorkspaceV4Schema;
 
-export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
+function parseWorkspaceDocument(source: string): unknown {
   const documents = parseAllDocuments(source, { uniqueKeys: true });
   if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
   const document = documents[0];
@@ -428,7 +375,48 @@ export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
     throw new Error(`Invalid workspace YAML: ${[...document.errors, ...document.warnings]
       .map((error) => error.message).join("; ")}`);
   }
-  return validateWorkspaceDescriptor(document.toJSON());
+  return document.toJSON();
+}
+
+export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
+  const value = parseWorkspaceDocument(source);
+  assertAuthoredWorkspaceIsDatabaseFree(value);
+  return validateWorkspaceDescriptor(value);
+}
+
+/** Parses an ephemeral, generated core runtime descriptor that may contain a Catalog binding. */
+export function parseRuntimeWorkspaceYaml(source: string): WorkspaceDescriptor {
+  return validateWorkspaceDescriptor(parseWorkspaceDocument(source));
+}
+
+function assertAuthoredWorkspaceIsDatabaseFree(workspace: unknown): void {
+  if (workspace !== null && typeof workspace === "object"
+    && ("dwh" in workspace || "diagnostics" in workspace)) {
+    throw new Error(
+      "Workspace YAML must not contain database configuration; use the PostgreSQL Metadata Catalog",
+    );
+  }
+}
+
+/** Build a database-free v4 descriptor; legacy database/configuration fields are not carried over. */
+export function migrateWorkspaceV3Yaml(source: string): string {
+  const documents = parseAllDocuments(source, { uniqueKeys: true });
+  if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
+  const document = documents[0];
+  if (document.errors.length > 0 || document.warnings.length > 0) {
+    throw new Error("Invalid workspace YAML");
+  }
+  const value = document.toJSON() as Record<string, unknown>;
+  const metadata = value.workspace as Record<string, unknown> | undefined;
+  if (!metadata || metadata.schema_version !== 3) {
+    throw new Error("Workspace migration requires schema version 3");
+  }
+  metadata.schema_version = 4;
+  delete value.dwh;
+  delete value.diagnostics;
+  delete value.semantic_index;
+  delete value.llm_policy;
+  return serializeWorkspaceYaml(validateWorkspaceDescriptor(value));
 }
 
 export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescriptor {
@@ -439,11 +427,11 @@ export function isCanonicalWorkspace(workspace: unknown): workspace is Canonical
   return WorkspaceDescriptorSchema.safeParse(workspace).success;
 }
 
-export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV3 {
+export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV4 {
   return WorkspaceDescriptorSchema.safeParse(workspace).success;
 }
 
-export function validateOperationalWorkspace(workspace: unknown): WorkspaceV3 {
+export function validateOperationalWorkspace(workspace: unknown): WorkspaceV4 {
   return validateWorkspaceDescriptor(workspace);
 }
 
@@ -461,6 +449,7 @@ export function resolveDiagnosticUrl(baseUrl: string, path: string): URL {
 
 export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string {
   const canonical = validateOperationalWorkspace(workspace);
+  assertAuthoredWorkspaceIsDatabaseFree(canonical);
   return stringify(canonical, { lineWidth: 0, sortMapEntries: true });
 }
 

backend/src/workspaces/secret-requirements.ts

@@ -101,7 +101,8 @@ function selectedTransport(
   descriptor: WorkspaceDescriptor,
   variables: readonly InstallationVariable[],
   env: NodeJS.ProcessEnv,
-): DwhTransport {
+): DwhTransport | undefined {
+  if (!descriptor.dwh) return undefined;
   const transportVariable = variables.find(({ role, suffix }) => role === "DWH" && suffix === "TRANSPORT");
   const value = transportVariable === undefined ? undefined : env[transportVariable.name];
   return isTransport(value) && descriptor.dwh.supported_transports.includes(value)
@@ -136,11 +137,14 @@ export function discoverWorkspaceSecretRequirements(
   const variables = buildInstallationContract(descriptor).variables;
   const transport = selectedTransport(descriptor, variables, env);
   const restHasNoAuthentication = transport === "rest_api" && descriptor.diagnostics?.dwh_rest?.auth === "none";
-  const requiredDwh = new Set(restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport]);
+  const requiredDwh = new Set(
+    transport === undefined || restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport],
+  );
 
   const requirements: WorkspaceSecretRequirement[] = [];
   for (const variable of variables) {
     if (variable.role === "DWH") {
+      if (transport === undefined) continue;
       if (variable.transports !== undefined && !variable.transports.includes(transport)) continue;
       const requirement = requirementFor(variable, requiredDwh.has(variable.suffix));
       if (requirement !== undefined && requirement.required) requirements.push(requirement);

backend/src/workspaces/types.ts

@@ -19,4 +19,4 @@ export type WorkspaceErrorCode =
   | "workspace_stale" | "git_unavailable" | "git_auth_failed" | "git_non_fast_forward"
   | "connector_unavailable" | "semantic_index_incompatible";
 
-export type { WorkspaceV3 } from "./schema.js";
+export type { WorkspaceV4 } from "./schema.js";

backend/src/workspaces/vector-collections.ts

@@ -0,0 +1,20 @@
+export interface WorkspaceVectorCollections {
+  reference: string;
+  memory: string;
+}
+
+/**
+ * Physical Qdrant namespaces owned by one workspace.
+ *
+ * Reference data is replaceable preprocessing output. Memory is durable runtime
+ * state and deliberately has a separate lifecycle.
+ */
+export function workspaceVectorCollections(workspaceId: string): WorkspaceVectorCollections {
+  if (!/^[a-z][a-z0-9-]{2,62}$/.test(workspaceId)) {
+    throw new Error("workspace id is invalid");
+  }
+  return {
+    reference: `${workspaceId}-reference`,
+    memory: `${workspaceId}-memory`,
+  };
+}

backend/test/archive-repair-response.test.ts

@@ -0,0 +1,48 @@
+import Fastify from "fastify";
+import { expect, test, vi } from "vitest";
+import { sessionRoutes } from "../src/routes/sessions.js";
+import type { PrincipalContext } from "../src/auth/principal.js";
+
+test.each([
+  [false, "m", 403], [false, "e", 403], [false, "reject", 204],
+  [true, "m", 204], [true, "e", 204], [true, "forged", 400],
+] as const)("archive repair response enforces the responding principal (%s, %s)", async (admin, choice, status) => {
+  const app = Fastify();
+  const principal: PrincipalContext = { issuer: "test", subject: "reviewer", isAdmin: admin,
+    roles: admin ? ["admin"] : ["user"],
+    permissions: admin ? ["session.use", "memory.manage", "evidence.manage"] : ["session.use"] };
+  app.addHook("preHandler", async request => { request.principal = principal; });
+  const respond = vi.fn(() => true);
+  sessionRoutes(app, {
+    tht: {}, mgr: { get: () => ({ ownerKey: "test\0reviewer", bridge: {
+      respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair", repair: { options: [
+        { id: "m", archive: "memory" }, { id: "e", archive: "evidence" },
+      ] } }),
+    } }) },
+  } as unknown as Parameters<typeof sessionRoutes>[1]);
+  try {
+    const result = await app.inject({ method: "POST", url: "/sessions/s/response",
+      payload: { ui_response: { id: "gate", choices: [choice] } } });
+    expect(result.statusCode).toBe(status);
+    expect(respond).toHaveBeenCalledTimes(status === 204 ? 1 : 0);
+  } finally { await app.close(); }
+});
+
+test("an administrator cannot attribute a repair to another runtime's principal", async () => {
+  const app = Fastify();
+  app.addHook("preHandler", async request => { request.principal = {
+    issuer: "test", subject: "second-admin", isAdmin: true, roles: ["admin"],
+    permissions: ["session.use", "memory.manage"],
+  }; });
+  const respond = vi.fn();
+  sessionRoutes(app, { tht: {}, mgr: { get: () => ({ ownerKey: "test\0first-admin", bridge: {
+    respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair",
+      repair: { options: [{ id: "m", archive: "memory" }] } }),
+  } }) } } as unknown as Parameters<typeof sessionRoutes>[1]);
+  try {
+    const result = await app.inject({ method: "POST", url: "/sessions/s/response",
+      payload: { ui_response: { id: "gate", choices: ["m"] } } });
+    expect(result.statusCode).toBe(409);
+    expect(respond).not.toHaveBeenCalled();
+  } finally { await app.close(); }
+});

backend/test/auth-config.test.ts

@@ -188,6 +188,8 @@ test("roles collapse duplicates and admin contains all administrative permission
     "workspace.manage",
     "workspace.secrets.manage",
     "database.manage",
+    "memory.manage",
+    "evidence.manage",
     "pi.manage",
     "auth.diagnostics.read",
   ]);

backend/test/auth-pi-management-local.test.ts

@@ -5,14 +5,12 @@ import { createLocalAuthFixture } from "./auth-test-fixtures.js";
 function fakeService(): PiManagementService {
   return {
     status: vi.fn(async () => ({ ready: true })),
-    options: vi.fn(async () => ({ providers: [], models: [], reasoning: [], checkedAt: "2026-08-17T00:00:00.000Z" })),
-    configure: vi.fn(async (value) => ({ ...value, updatedAt: "2026-08-17T00:00:00.000Z" })),
     test: vi.fn(async () => ({ ready: true, checkedAt: "2026-08-17T00:00:00.000Z" })),
     logs: vi.fn(async () => ({ lines: [] })),
   };
 }
 
-test("a local HTTPS cookie session authorizes Pi writes through an untrusted internal HTTP hop", async () => {
+test("a local HTTPS cookie session authorizes the Pi smoke check through an untrusted internal HTTP hop", async () => {
   const service = fakeService();
   const fixture = await createLocalAuthFixture(
     { piManagement: service },
@@ -25,31 +23,21 @@ test("a local HTTPS cookie session authorizes Pi writes through an untrusted int
     expect(fixture.publicUrl).toBe("HTTPS://thothii.example.test");
 
     const proxyHeaders = fixture.sessionHeaders({ host: "127.0.0.1:8080" });
-    const configured = await fixture.app.inject({
-      method: "PUT",
-      url: "/pi-management/config",
-      headers: proxyHeaders,
-      payload: { provider: "zai", model: "glm-5.2", reasoning: "high" },
-    });
     const smoke = await fixture.app.inject({
       method: "POST",
       url: "/pi-management/test",
       headers: proxyHeaders,
     });
 
-    expect(configured.statusCode).toBe(200);
     expect(smoke.statusCode).toBe(200);
-    expect(service.configure).toHaveBeenCalledTimes(1);
     expect(service.test).toHaveBeenCalledTimes(1);
 
     fixture.resetDownstreamHits();
-    vi.mocked(service.configure).mockClear();
     vi.mocked(service.test).mockClear();
     const wrongOrigin = await fixture.app.inject({
-      method: "PUT",
-      url: "/pi-management/config",
+      method: "POST",
+      url: "/pi-management/test",
       headers: fixture.sessionHeaders({ host: "127.0.0.1:8080", origin: "https://evil.example" }),
-      payload: { provider: "zai", model: "glm-5.2", reasoning: "high" },
     });
     const wrongCsrf = await fixture.app.inject({
       method: "POST",
@@ -62,7 +50,6 @@ test("a local HTTPS cookie session authorizes Pi writes through an untrusted int
       expect(response.json()).toEqual({ code: "csrf_failed", error: "Request origin validation failed" });
     }
     expect(fixture.downstreamHits()).toBe(0);
-    expect(service.configure).not.toHaveBeenCalled();
     expect(service.test).not.toHaveBeenCalled();
   } finally {
     await fixture.close();

backend/test/auth-routes-local.test.ts

@@ -130,7 +130,7 @@ test("local login sets a non-persistent opaque session cookie and exposes only a
     roles: ["admin"],
     permissions: [
       "session.use", "session.read_all", "session.manage_all", "settings.manage",
-      "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
+      "workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
     ],
     isAdmin: true,
     csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/),
@@ -416,19 +416,6 @@ test("only two Argon2 verifications run concurrently and excess login attempts f
   expect((await second).statusCode).toBe(401);
 });
 
-test("the real native asynchronous Argon2 verifier holds two permits and releases them after completion", async () => {
-  const { app } = await createLocalApp();
-  const first = login(app, { password: `${password}!` });
-  const second = login(app, { password: `${password}!` });
-  await new Promise<void>((resolve) => setImmediate(resolve));
-
-  const excess = await login(app, { password: `${password}!` });
-  expect(excess.statusCode).toBe(429);
-  await expect(first).resolves.toMatchObject({ statusCode: 401 });
-  await expect(second).resolves.toMatchObject({ statusCode: 401 });
-  await expect(login(app, { password: `${password}!` })).resolves.toMatchObject({ statusCode: 401 });
-});
-
 test("a verifier failure is sanitized and releases its concurrency permit", async () => {
   let attempts = 0;
   const user = {

backend/test/auth.test.ts

@@ -7,6 +7,8 @@ import { join } from "node:path";
 import { expandLocalHome, localPrincipal, upstreamPrincipal } from "../src/auth/principal.js";
 import { buildApp } from "../src/app.js";
 import { loadConfig } from "../src/config.js";
+import { rolesToPermissions } from "../src/auth/config.js";
+import type { Permission, Role } from "../src/auth/types.js";
 
 test("server smoke rejects retired trusted claims under OIDC authentication", () => {
   const smoke = readFileSync("../scripts/unified-deployment-smoke.sh", "utf8");
@@ -32,7 +34,7 @@ test("local mode resolves a stable local principal", async () => {
     roles: ["admin"],
     permissions: [
       "session.use", "session.read_all", "session.manage_all", "settings.manage",
-      "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
+      "workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
     ],
     isAdmin: true,
   });
@@ -74,7 +76,7 @@ test("upstream mode accepts only normalized proxy principal headers", async () =
     issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"],
     permissions: [
       "session.use", "session.read_all", "session.manage_all", "settings.manage",
-      "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
+      "workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
     ],
     isAdmin: true,
   });
@@ -206,15 +208,33 @@ test("the session boundary rejects tht maintenance headers outside exact loopbac
   })).statusCode).toBe(503);
 });
 
-test("the session boundary touches a valid cookie session through the bounded Task 7 store operation", async () => {
+test.each<{
+  name: string;
+  roles: Role[];
+  storedPermissions: Permission[];
+}>([
+  { name: "current user", roles: ["user"], storedPermissions: ["session.use"] },
+  {
+    name: "administrator signed in before archive permissions existed",
+    roles: ["admin"],
+    storedPermissions: rolesToPermissions(["admin"]).filter(
+      (permission) => permission !== "memory.manage" && permission !== "evidence.manage",
+    ),
+  },
+  {
+    name: "user with obsolete administrative permissions",
+    roles: ["user"],
+    storedPermissions: ["session.use", "memory.manage", "evidence.manage"],
+  },
+])("the session boundary derives current permissions and touches a valid cookie: $name", async ({ roles, storedPermissions }) => {
   const sessions = {
     resolve: vi.fn(async () => ({
       version: 1,
       issuer: "local",
       subject: "user-1",
       method: "local",
-      roles: ["user"],
-      permissions: ["session.use"],
+      roles,
+      permissions: storedPermissions,
       userAuthRevision: 1,
       authConfigRevision: "b".repeat(64),
       remembered: false,
@@ -249,7 +269,13 @@ test("the session boundary touches a valid cookie session through the bounded Ta
   app.get("/private", async (request) => getPrincipal(request));
 
   const token = "z".repeat(43);
-  expect((await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } })).statusCode).toBe(200);
+  const response = await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } });
+  expect(response.statusCode).toBe(200);
+  expect(response.json()).toMatchObject({
+    roles,
+    permissions: rolesToPermissions(roles),
+    isAdmin: roles.includes("admin"),
+  });
   expect(sessions.touch).toHaveBeenCalledWith(token);
 });
 

backend/test/catalog-databases-routes.test.ts

@@ -11,6 +11,7 @@ import type { ObservedSchemaSnapshot } from "../src/catalog/types.js";
 import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
 import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
+import type { WorkspaceDiagnoser } from "../src/routes/workspaces.js";
 
 const roots: string[] = [];
 afterEach(() => {
@@ -19,16 +20,11 @@ afterEach(() => {
 });
 
 const workspace: WorkspaceDescriptor = {
-  workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
+  workspace: { schema_version: 4, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
   dwh: {
     engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432,
     supported_transports: ["postgres_direct", "rest_api"],
   },
-  semantic_index: {
-    vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
-    embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-  },
-  llm_policy: { allowed: ["zai/glm-5.2"] },
   diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } },
 };
 const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
@@ -38,6 +34,7 @@ function setup(
   catalogDependencies: {
     catalogOperationCoordinator?: CatalogOperationCoordinator;
     catalogPostgresAccess?: CatalogPostgresAccess;
+    workspaceDiagnoser?: WorkspaceDiagnoser;
   } = {},
   workspaceDescriptor: WorkspaceDescriptor = workspace,
 ) {
@@ -61,7 +58,7 @@ function setup(
     workspaceRegistry: registry,
     workspaceSecretStore: secretStore,
     catalogRepository: repository,
-    workspaceDiagnoser: vi.fn(),
+    workspaceDiagnoser: vi.fn(async () => ({ activatable: true, diagnostics: [] })),
     ...catalogDependencies,
   });
   return { app, secretStore, repository };
@@ -100,31 +97,23 @@ const fleetSnapshot: ObservedSchemaSnapshot = {
   }],
 };
 
-test("lists every YAML workspace and creates its one database configuration", async () => {
+test("lists every workspace and creates its Catalog database configuration", async () => {
   const { app, secretStore } = setup();
   const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
   expect(initial.statusCode).toBe(200);
   expect(initial.json()).toMatchObject([{
     workspaceId: "psd-clinical",
     configured: false,
-    databaseName: "warehouse",
+    databaseName: "",
     workspaceRevision: { commit: revision.commit, blob: revision.blob },
     workspaceEvidence: { sourceType: null, state: "not_declared" },
-    runtimeBinding: {
-      transport: "postgres_direct",
-      configurationState: "configuration_required",
-      sessionTransportSupported: true,
-    },
+    runtimeBinding: null,
   }]);
 
-  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal");
-  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
-  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader");
-  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
   secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" });
   const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" });
   expect(runtimeReady.json()).toMatchObject([{
-    runtimeBinding: { configurationState: "ready", sessionTransportSupported: true },
+    runtimeBinding: null,
   }]);
 
   const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
@@ -169,7 +158,7 @@ test("projects remote Evidence credential state without conflating catalog secre
   }]);
 });
 
-test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
+test("lists orphaned records and keeps the REST diagnostic path in the Catalog", async () => {
   const { app, repository } = setup();
   await repository.create({
     workspaceId: "removed-workspace",
@@ -189,7 +178,7 @@ test("lists orphaned records and takes the REST diagnostic path from workspace Y
     },
   });
   expect(created.statusCode).toBe(201);
-  expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
+  expect(created.json()).toMatchObject({ binding: { restPath: "/client-controlled" } });
 
   const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
   expect(rows).toEqual(expect.arrayContaining([
@@ -280,6 +269,72 @@ test("rejects a connection test while another catalog operation owns the databas
   }
 });
 
+test("workspace and database tests use the same current catalog database binding", async () => {
+  const connect = vi.fn(async () => ({
+    query: vi.fn(async () => ({
+      rows: [{ database: "warehouse", schema: "datawarehouse" }],
+    })),
+    end: vi.fn(async () => undefined),
+  }));
+  const diagnose: WorkspaceDiagnoser = vi.fn(async () => ({
+    activatable: true,
+    diagnostics: [{
+      level: "info",
+      code: "binding_ok",
+      message: "Installation bindings and diagnostics succeeded.",
+    }],
+  }));
+  const { app } = setup({
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
+    THT_WS_PSD_CLINICAL_DWH_HOST: "legacy-db.internal",
+    THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
+    THT_WS_PSD_CLINICAL_DWH_USER: "legacy-reader",
+  }, {
+    catalogPostgresAccess: { connect } as CatalogPostgresAccess,
+    workspaceDiagnoser: diagnose,
+  });
+  const created = (await app.inject({
+    method: "POST",
+    url: "/catalog/databases",
+    payload: {
+      ...direct,
+      binding: { ...direct.binding, host: "current-db.internal", username: "current-reader" },
+    },
+  })).json();
+
+  const databaseTest = await app.inject({
+    method: "POST",
+    url: `/catalog/databases/${created.id}/test`,
+    payload: { version: created.version },
+  });
+  const workspaceTest = await app.inject({
+    method: "POST",
+    url: "/workspaces/psd-clinical/test",
+    payload: {},
+  });
+
+  expect(databaseTest.statusCode).toBe(200);
+  expect(workspaceTest.statusCode).toBe(200);
+  expect(connect).toHaveBeenCalledTimes(2);
+  expect(connect.mock.calls.map(([database]) => database)).toEqual([
+    expect.objectContaining({
+      databaseName: "warehouse",
+      schema: "datawarehouse",
+      binding: expect.objectContaining({ host: "current-db.internal", username: "current-reader" }),
+    }),
+    expect.objectContaining({
+      databaseName: "warehouse",
+      schema: "datawarehouse",
+      binding: expect.objectContaining({ host: "current-db.internal", username: "current-reader" }),
+    }),
+  ]);
+  expect(diagnose).toHaveBeenCalledWith(
+    workspace,
+    expect.any(Object),
+    { writeProbe: false, skipDwh: true },
+  );
+});
+
 test("returns exact global and per-database fleet metrics", async () => {
   const { app, repository } = setup();
   const database = await repository.create(direct);

backend/test/catalog-description-generation-routes.test.ts

@@ -14,6 +14,7 @@ import {
   type ModelCompletionRequest,
 } from "../src/catalog/model-completer.js";
 import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
+import type { SensitivityValueSource } from "../src/catalog/sensitivity-classifier.js";
 import type {
   CatalogDatabaseClient,
   CatalogPostgresAccess,
@@ -24,7 +25,7 @@ import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
 
 const workspace: WorkspaceDescriptor = {
   workspace: {
-    schema_version: 3,
+    schema_version: 4,
     id: "psd-clinical",
     name: "Policlinico San Donato",
     language: "it",
@@ -36,11 +37,6 @@ const workspace: WorkspaceDescriptor = {
     port: 5432,
     supported_transports: ["postgres_direct"],
   },
-  semantic_index: {
-    vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
-    embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
-  },
-  llm_policy: { allowed: ["zai/glm-5.2"] },
 };
 const revision: WorkspaceRevision = {
   id: "psd-clinical",
@@ -49,7 +45,7 @@ const revision: WorkspaceRevision = {
   snapshotPath: "/tmp/psd.yaml",
 };
 const configuredModel: ResolvedMetadataGenerationModel = {
-  id: "openai-mini",
+  id: "openai/gpt-4.1-mini",
   provider: "openai",
   model: "gpt-4.1-mini",
   apiKeyEnv: "OPENAI_API_KEY",
@@ -74,6 +70,16 @@ async function setup(
     sample: vi.fn(async () => []),
   },
   catalogPostgresAccess?: CatalogPostgresAccess,
+  sensitivityValueSource: SensitivityValueSource = {
+    scanTable: vi.fn(async (request, consume) => {
+      await consume(request.columns.map((column) => ({
+        columnId: column.id,
+        value: "ordinary",
+        characterLength: 8,
+      })));
+      return { kind: "complete", observedValues: 1 };
+    }),
+  },
 ) {
   const repository = new MemoryCatalogRepository();
   const database = await repository.create({
@@ -120,10 +126,11 @@ async function setup(
     catalogOperationCoordinator: operations,
     metadataGenerationModels: models(),
     modelCompleter,
+    sensitivityValueSource,
     ...(descriptionSourceSampler ? { descriptionSourceSampler } : {}),
     ...(catalogPostgresAccess ? { catalogPostgresAccess } : {}),
   });
-  return { app, repository, database, table, column, operations };
+  return { app, repository, database, table, column, operations, sensitivityValueSource };
 }
 
 async function waitForTerminalRun(app: ReturnType<typeof buildApp>, runId: string) {
@@ -141,22 +148,15 @@ async function waitForTerminalRun(app: ReturnType<typeof buildApp>, runId: strin
   throw new Error(`Description Generation Run ${runId} did not finish`);
 }
 
-test("suggests sensitive flags from structural metadata without persisting them", async () => {
-  const modelCompleter = {
-    complete: vi.fn(async () => JSON.stringify({
-      suggestions: [{ columnId: expect.any(String), sensitive: true }],
-    })),
-  };
+test("assesses sensitive flags locally without persisting them or calling an LLM", async () => {
+  const modelCompleter: ModelCompleter = { complete: vi.fn(async () => "unused") };
   const { app, repository, database, table, column } = await setup(modelCompleter);
-  modelCompleter.complete.mockResolvedValueOnce(JSON.stringify({
-    suggestions: [{ columnId: column.id, sensitive: true }],
-  }));
 
   try {
     const response = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { modelId: configuredModel.id, scope: "all" },
+      payload: { scope: "all" },
     });
 
     expect(response.statusCode).toBe(200);
@@ -165,11 +165,14 @@ test("suggests sensitive flags from structural metadata without persisting them"
       run: {
         databaseId: database.id,
         scope: "all",
-        modelId: configuredModel.id,
+        engine: "local",
+        modelId: null,
+        policyVersion: "sensitivity-v4",
         status: "completed",
         total: 1,
         suggestedSensitive: 1,
         suggestedNonSensitive: 0,
+        unknown: 0,
         errorSummary: null,
       },
       suggestions: [{
@@ -180,6 +183,9 @@ test("suggests sensitive flags from structural metadata without persisting them"
         version: column.version,
         currentSensitive: false,
         sensitive: true,
+        assessment: "sensitive",
+        evidence: [{ kind: "metadata", ruleId: "metadata.direct_identifier" }],
+        coverage: "metadata",
       }],
     });
     expect(await repository.getColumn(database.id, column.tableId, column.id))
@@ -212,49 +218,69 @@ test("suggests sensitive flags from structural metadata without persisting them"
         runId: responseBody.run.id,
         sequence: 1,
         level: "info",
-        message: "Sensitive-field suggestion generation started.",
+        message: "Local sensitivity analysis started.",
       },
       {
         runId: responseBody.run.id,
         sequence: 2,
         level: "info",
-        message: "Classified 1 of 1 columns.",
+        message: "Scanning source data: pass 1 of 3, table batch 1 of 1.",
       },
       {
         runId: responseBody.run.id,
         sequence: 3,
         level: "info",
-        message: "Sensitive-field suggestion generation completed for 1 column.",
+        message: "Scanning source data: pass 2 of 3, table batch 1 of 1.",
+      },
+      {
+        runId: responseBody.run.id,
+        sequence: 4,
+        level: "info",
+        message: "Scanning source data: pass 3 of 3, table batch 1 of 1.",
+      },
+      {
+        runId: responseBody.run.id,
+        sequence: 5,
+        level: "info",
+        message: "Assessed 1 of 1 columns locally.",
+      },
+      {
+        runId: responseBody.run.id,
+        sequence: 6,
+        level: "info",
+        message: "Local sensitivity analysis completed for 1 column.",
       },
     ]);
 
-    const request = modelCompleter.complete.mock.calls[0]![0] as ModelCompletionRequest;
-    const prompt = request.messages.map((message) => message.content).join("\n");
-    expect(prompt).toContain("patients");
-    expect(prompt).toContain("birth_date");
-    expect(prompt).toContain("date");
-    expect(prompt).not.toContain("Patient date of birth");
-    expect(prompt).not.toContain("test-provider-secret");
+    expect(modelCompleter.complete).not.toHaveBeenCalled();
   } finally {
     await app.close();
   }
 });
 
-test("limits sensitive-data suggestions to the selected tables or columns", async () => {
-  const modelCompleter: ModelCompleter = {
-    complete: vi.fn(async (request) => {
-      const payload = JSON.parse(request.messages.find((message) => message.role === "user")!.content) as {
-        columns: Array<{ columnId: string; column: string }>;
-      };
-      return JSON.stringify({
-        suggestions: payload.columns.map((column) => ({
-          columnId: column.columnId,
-          sensitive: column.column.includes("name") || column.column.includes("note"),
-        })),
-      });
-    }),
-  };
-  const { app, repository, database } = await setup(modelCompleter);
+test("does not impose a global HTTP deadline on sensitivity analysis", async () => {
+  const timeout = vi.spyOn(AbortSignal, "timeout");
+  const { app, repository, database } = await setup({ complete: vi.fn(async () => "unused") });
+
+  try {
+    const response = await app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
+      payload: { scope: "all" },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(timeout).not.toHaveBeenCalled();
+    expect(await repository.listSensitivityAnalysisRuns()).toHaveLength(1);
+  } finally {
+    timeout.mockRestore();
+    await app.close();
+  }
+});
+
+test("limits sensitivity analysis to the selected tables or columns", async () => {
+  const modelCompleter: ModelCompleter = { complete: vi.fn(async () => "unused") };
+  const { app, repository, database, sensitivityValueSource } = await setup(modelCompleter);
   await repository.applySchemaSync(database.id, database.version, "all", [], {
     schemaVersion: 1,
     capabilities: { tables: "available", columns: "available", relationships: "available" },
@@ -286,7 +312,6 @@ test("limits sensitive-data suggestions to the selected tables or columns", asyn
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
-        modelId: configuredModel.id,
         scope: "selected_tables",
         targetIds: [visits.id, patients.id],
       },
@@ -306,7 +331,6 @@ test("limits sensitive-data suggestions to the selected tables or columns", asyn
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
-        modelId: configuredModel.id,
         scope: "selected_columns",
         targetIds: [clinicalNote.id, status.id],
       },
@@ -318,44 +342,41 @@ test("limits sensitive-data suggestions to the selected tables or columns", asyn
       expect.objectContaining({ tableId: visits.id, columnId: clinicalNote.id, sensitive: true }),
     ]));
 
-    const prompts = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => (
-      JSON.parse(request.messages.find((message) => message.role === "user")!.content) as {
-        columns: Array<{ columnId: string }>;
-      }
-    ));
-    expect(prompts[0]!.columns.map((column) => column.columnId).sort()).toEqual(
-      [...patientColumns, ...visitColumns].map((column) => column.id).sort(),
-    );
-    expect(prompts[0]!.columns.map((column) => column.columnId)).not.toContain(billingColumns[0]!.id);
-    expect(prompts[1]!.columns.map((column) => column.columnId).sort()).toEqual(
-      [status.id, clinicalNote.id].sort(),
+    const scannedColumnIds = vi.mocked(sensitivityValueSource.scanTable).mock.calls.flatMap(
+      ([request]) => request.columns.map((column) => column.id),
     );
+    expect(scannedColumnIds).toEqual([status.id, status.id]);
+    expect(scannedColumnIds).not.toContain(patientColumns.find(
+      (column) => column.name === "patient_name",
+    )!.id);
+    expect(scannedColumnIds).not.toContain(clinicalNote.id);
+    expect(scannedColumnIds).not.toContain(billingColumns[0]!.id);
+    expect(modelCompleter.complete).not.toHaveBeenCalled();
   } finally {
     await app.close();
   }
 });
 
-test("explains invalid sensitive-data suggestion selections without calling the model", async () => {
+test("explains invalid sensitivity-analysis selections without reading source values", async () => {
   const modelCompleter: ModelCompleter = { complete: vi.fn(async () => "unused") };
-  const { app, database, table } = await setup(modelCompleter);
+  const { app, database, table, sensitivityValueSource } = await setup(modelCompleter);
 
   try {
     const empty = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { modelId: configuredModel.id, scope: "selected_tables", targetIds: [] },
+      payload: { scope: "selected_tables", targetIds: [] },
     });
     expect(empty.statusCode).toBe(400);
     expect(empty.json()).toEqual({
       code: "sensitive_data_suggestion_request_invalid",
-      message: "Choose a database, one or more tables, or one or more columns to classify.",
+      message: "Choose a database, one or more tables, or one or more columns to assess.",
     });
 
     const duplicate = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
-        modelId: configuredModel.id,
         scope: "selected_tables",
         targetIds: [table.id, table.id],
       },
@@ -370,7 +391,6 @@ test("explains invalid sensitive-data suggestion selections without calling the
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
-        modelId: configuredModel.id,
         scope: "selected_tables",
         targetIds: ["00000000-0000-4000-8000-000000000001"],
       },
@@ -385,7 +405,6 @@ test("explains invalid sensitive-data suggestion selections without calling the
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
-        modelId: configuredModel.id,
         scope: "selected_columns",
         targetIds: ["00000000-0000-4000-8000-000000000002"],
       },
@@ -396,205 +415,7 @@ test("explains invalid sensitive-data suggestion selections without calling the
       message: "One or more selected Catalog Columns were not found in this database.",
     });
     expect(modelCompleter.complete).not.toHaveBeenCalled();
-  } finally {
-    await app.close();
-  }
-});
-
-test("batches sensitive-data suggestions for schemas larger than one helper message", async () => {
-  const maxHelperMessageBytes = 64 * 1024;
-  const seenColumnIds: string[] = [];
-  const modelCompleter: ModelCompleter = {
-    complete: vi.fn(async (request) => {
-      const userMessage = request.messages.find((message) => message.role === "user")!;
-      expect(Buffer.byteLength(userMessage.content, "utf8")).toBeLessThanOrEqual(maxHelperMessageBytes);
-      const payload = JSON.parse(userMessage.content) as {
-        columns: Array<{ columnId: string; column: string }>;
-      };
-      expect(payload.columns.length).toBeLessThanOrEqual(10);
-      seenColumnIds.push(...payload.columns.map((column) => column.columnId));
-      return JSON.stringify({
-        suggestions: payload.columns.map((column) => ({
-          columnId: column.columnId,
-          sensitive: column.column.endsWith("_private"),
-        })),
-      });
-    }),
-  };
-  const { app, repository, database } = await setup(modelCompleter);
-  const columnCount = 900;
-  await repository.applySchemaSync(database.id, database.version, "all", [], {
-    schemaVersion: 1,
-    capabilities: { tables: "available", columns: "available", relationships: "available" },
-    tables: [{ name: "wide_table", sourceComment: null }],
-    columns: Array.from({ length: columnCount }, (_, index) => ({
-      tableName: "wide_table",
-      name: `field_${index.toString().padStart(4, "0")}${index % 10 === 0 ? "_private" : ""}`,
-      ordinalPosition: index + 1,
-      dataType: "character varying(255)",
-      isNullable: true,
-      defaultExpression: null,
-      primaryKeyPosition: null,
-      sourceComment: null,
-    })),
-    relationships: [],
-  });
-  const wideTable = (await repository.listTables(database.id)).find((table) => table.name === "wide_table")!;
-  const expectedColumnIds = (await repository.listColumns(database.id, wideTable.id)).map((column) => column.id);
-
-  try {
-    const response = await app.inject({
-      method: "POST",
-      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { modelId: configuredModel.id, scope: "all" },
-    });
-
-    expect(response.statusCode).toBe(200);
-    const suggestions = response.json().suggestions as Array<{
-      columnName: string;
-      currentSensitive: boolean;
-      sensitive: boolean;
-    }>;
-    expect(suggestions).toHaveLength(columnCount);
-    expect(suggestions).toEqual(expect.arrayContaining([
-      expect.objectContaining({ columnName: "field_0000_private", currentSensitive: false, sensitive: true }),
-      expect.objectContaining({ columnName: "field_0001", currentSensitive: false, sensitive: false }),
-    ]));
-    expect(vi.mocked(modelCompleter.complete).mock.calls.length).toBeGreaterThan(1);
-    expect(seenColumnIds.slice().sort()).toEqual(expectedColumnIds.slice().sort());
-    expect(new Set(seenColumnIds).size).toBe(columnCount);
-  } finally {
-    await app.close();
-  }
-});
-
-test("retries one invalid sensitive-data classification before returning the review draft", async () => {
-  const modelCompleter: ModelCompleter = {
-    complete: vi.fn(async () => "unused"),
-  };
-  const { app, database, column } = await setup(modelCompleter);
-  vi.mocked(modelCompleter.complete)
-    .mockResolvedValueOnce("not-json")
-    .mockResolvedValueOnce(JSON.stringify({
-      suggestions: [{ columnId: column.id, sensitive: true }],
-    }));
-
-  try {
-    const response = await app.inject({
-      method: "POST",
-      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { modelId: configuredModel.id, scope: "all" },
-    });
-
-    expect(response.statusCode).toBe(200);
-    expect(response.json().suggestions).toEqual([
-      expect.objectContaining({ columnId: column.id, sensitive: true }),
-    ]);
-    expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
-  } finally {
-    await app.close();
-  }
-});
-
-test.each(["malformed", "incomplete", "duplicate"] as const)(
-  "fails safely when sensitive-data suggestions are %s",
-  async (kind) => {
-    const modelCompleter: ModelCompleter = {
-      complete: vi.fn(async () => "unused"),
-    };
-    const { app, repository, database, column } = await setup(modelCompleter);
-    const rawResponse = kind === "malformed"
-      ? "RAW_PROVIDER_RESPONSE_DO_NOT_EXPOSE_{"
-      : kind === "incomplete"
-        ? JSON.stringify({ suggestions: [] })
-        : JSON.stringify({
-          suggestions: [
-            { columnId: column.id, sensitive: true },
-            { columnId: column.id, sensitive: true },
-          ],
-        });
-    vi.mocked(modelCompleter.complete).mockResolvedValueOnce(rawResponse);
-
-    try {
-      const response = await app.inject({
-        method: "POST",
-        url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-        payload: { modelId: configuredModel.id, scope: "all" },
-      });
-
-      expect(response.statusCode).toBe(502);
-      expect(response.json()).toEqual({
-        code: "sensitive_data_suggestion_invalid_response",
-        message: "The LLM returned an incomplete or invalid classification. No suggestions were applied.",
-      });
-      expect(response.body).not.toContain(rawResponse);
-      expect(await repository.getColumn(database.id, column.tableId, column.id))
-        .toMatchObject({ sensitive: false });
-    } finally {
-      await app.close();
-    }
-  },
-);
-
-test("explains a sensitive-data suggestion provider failure without exposing provider details", async () => {
-  const modelCompleter: ModelCompleter = {
-    complete: vi.fn(async () => {
-      throw new ModelCompletionProviderError();
-    }),
-  };
-  const { app, repository, database, column } = await setup(modelCompleter);
-
-  try {
-    const response = await app.inject({
-      method: "POST",
-      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { modelId: configuredModel.id, scope: "all" },
-    });
-
-    expect(response.statusCode).toBe(502);
-    expect(response.json()).toEqual({
-      code: "sensitive_data_suggestion_provider_unavailable",
-      message: "The selected LLM service could not complete the request. No suggestions were applied.",
-    });
-    expect(response.body).not.toContain("model completion failed");
-    expect(await repository.getColumn(database.id, column.tableId, column.id))
-      .toMatchObject({ sensitive: false });
-
-    const history = await app.inject({
-      method: "GET",
-      url: "/catalog/sensitive-data-suggestion-runs",
-    });
-    expect(history.statusCode).toBe(200);
-    const [failedRun] = history.json();
-    expect(failedRun).toMatchObject({
-      databaseId: database.id,
-      status: "failed",
-      total: 1,
-      suggestedSensitive: 0,
-      suggestedNonSensitive: 0,
-      errorSummary: "Sensitive-field suggestion generation failed.",
-    });
-
-    const events = await app.inject({
-      method: "GET",
-      url: `/catalog/sensitive-data-suggestion-runs/${failedRun.id}/events-list`,
-    });
-    expect(events.statusCode).toBe(200);
-    expect(events.json()).toMatchObject([
-      {
-        runId: failedRun.id,
-        sequence: 1,
-        level: "info",
-        message: "Sensitive-field suggestion generation started.",
-      },
-      {
-        runId: failedRun.id,
-        sequence: 2,
-        level: "error",
-        message: "Sensitive-field suggestion generation failed.",
-      },
-    ]);
-    expect(events.body).not.toContain("model completion failed");
+    expect(sensitivityValueSource.scanTable).not.toHaveBeenCalled();
   } finally {
     await app.close();
   }
@@ -705,7 +526,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
     expect(completionRequest.messages[0]?.content).toContain('{"results":[');
     expect(completionRequest.messages[1]?.content).toContain(`"targetId":"${column.id}"`);
     expect(completionRequest.messages[1]?.content).not.toMatch(/source rows|samples|example values/i);
-    expect(start.body).not.toMatch(/test-provider-secret|gpt-4\.1|openai\/gpt|Catalog metadata/);
+    expect(start.body).not.toMatch(/test-provider-secret|Catalog metadata/);
 
     resolveCompletion(`\`\`\`json\n${JSON.stringify({
       results: [{
@@ -756,7 +577,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
       expect.objectContaining({
         sequence: 3,
         level: "info",
-        message: `Generated description for Catalog Column ${column.id}.`,
+        message: 'Generated description for Column "patients.birth_date".',
       }),
       expect.objectContaining({ sequence: 4, level: "info", message: "Description generation completed." }),
     ]);
@@ -1113,7 +934,10 @@ test("generates selected Catalog Columns in caller order through sequential batc
     const events = await repository.listDescriptionGenerationEvents(run.id);
     expect(events.map((event) => event.sequence)).toEqual(Array.from({ length: 14 }, (_, index) => index + 1));
     expect(events.slice(2, -1).map((event) => event.message)).toEqual(
-      orderedIds.map((targetId) => `Generated description for Catalog Column ${targetId}.`),
+      orderedIds.map((targetId) => {
+        const target = columns.find((column) => column.id === targetId)!;
+        return `Generated description for Column "patients.${target.name}".`;
+      }),
     );
   } finally {
     pending[0]!.resolve(responseFor(orderedIds.slice(0, 10), 0));
@@ -1982,7 +1806,9 @@ test("retains completed batch writes when a later batch response is malformed",
     });
     const events = await repository.listDescriptionGenerationEvents(run.id);
     expect(events.slice(2, 12).map((event) => event.message)).toEqual(
-      orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
+      originalColumns.slice(0, 10).map(
+        (target) => `Generated description for Column "patients.${target.name}".`,
+      ),
     );
     expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
       expect.objectContaining({
@@ -1991,7 +1817,7 @@ test("retains completed batch writes when a later batch response is malformed",
     );
     expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
       level: "error",
-      message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
+      message: `The model response did not match the required schema. Affected target: Column "patients.${originalColumns[10]!.name}".`,
     }));
   } finally {
     await app.close();
@@ -2480,7 +2306,7 @@ test("generates selected Catalog Tables with structural column context and local
     expect((await repository.listDescriptionGenerationEvents(run.id)).map((event) => event.message)).toEqual([
       "Description generation queued.",
       "Description generation started.",
-      `Stored non-generatable result for Catalog Table ${table.id}.`,
+      'Stored non-generatable result for Table "patients".',
       "Description generation completed.",
     ]);
   } finally {
@@ -2624,7 +2450,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
     expect(`${JSON.stringify(run)}${events.body}`).not.toContain(sensitiveDiagnostic);
     expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
       level: "error",
-      message: `The model provider request failed. Affected Catalog Column target: ${column.id}.`,
+      message: 'The model provider request failed. Affected target: Column "patients.birth_date".',
     }));
   } finally {
     await app.close();
@@ -2715,7 +2541,7 @@ test.each([
     expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
       expect.objectContaining({
         level: "error",
-        message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
+        message: `${failureMessage} Affected targets: Column "patients.first_column", Column "patients.second_column".`,
       }),
     );
   } finally {
@@ -2909,7 +2735,7 @@ test("validates selected targets and resolves every requested target before laun
     const unknownModel = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "unknown-model", scope: "selected_columns", targetIds: [column.id] },
+      payload: { modelId: "openai/unknown-model", scope: "selected_columns", targetIds: [column.id] },
     });
     expect(unknownModel.statusCode).toBe(409);
     expect(unknownModel.json().code).toBe("metadata_generation_model_unavailable");

backend/test/catalog-description-generation.integration.test.ts

@@ -13,7 +13,12 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
 import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
 import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
 import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
+import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
 import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
+import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
+import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
+import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
+import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
 import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
 import { loadConfig } from "../src/config.js";
 import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
@@ -49,7 +54,12 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     await upSensitiveDataFlag(db);
     await upDescriptionGeneration(db);
     await upSensitiveSuggestionRuns(db);
+    await upLogicalRelationships(db);
     await upAiTokenUsage(db);
+    await upCanonicalModelIds(db);
+    await upLocalSensitivityAnalysis(db);
+    await upSensitivityReason(db);
+    await upCatalogPreprocessingState(db);
     const repository = new KyselyCatalogRepository(db);
     const database = await repository.create({
       workspaceId: "psd-clinical",
@@ -158,9 +168,9 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
       }),
     };
     const models: MetadataGenerationModels = {
-      catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
+      catalog: () => ({ models: [{ id: "openai/gpt-4.1-mini", label: "OpenAI Mini" }], default: "openai/gpt-4.1-mini" }),
       resolve: () => ({
-        id: "openai-mini",
+        id: "openai/gpt-4.1-mini",
         provider: "openai",
         model: "gpt-4.1-mini",
         apiKeyEnv: "OPENAI_API_KEY",
@@ -205,12 +215,12 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
       payload: {
-        modelId: "openai-mini",
+        modelId: "openai/gpt-4.1-mini",
         scope: "selected_columns",
         targetIds: [status.id, birthDate.id],
       },
     });
-    expect(successfulStart.statusCode).toBe(202);
+    expect(successfulStart.statusCode, successfulStart.body).toBe(202);
     expect(await terminalRun(app, successfulStart.json().id)).toMatchObject({
       status: "completed",
       total: 2,
@@ -233,7 +243,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     const tableStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai-mini", scope: "selected_tables", targetIds: [table.id] },
+      payload: { modelId: "openai/gpt-4.1-mini", scope: "selected_tables", targetIds: [table.id] },
     });
     expect(tableStart.statusCode).toBe(202);
     expect(await terminalRun(app, tableStart.json().id)).toMatchObject({
@@ -253,7 +263,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     const failedStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai-mini", scope: "selected_columns", targetIds: [status.id] },
+      payload: { modelId: "openai/gpt-4.1-mini", scope: "selected_columns", targetIds: [status.id] },
     });
     expect(failedStart.statusCode).toBe(202);
     const failedRun = await terminalRun(app, failedStart.json().id);
@@ -276,14 +286,14 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     expect(events.statusCode).toBe(200);
     expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
       level: "error",
-      message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
+      message: 'The model provider request failed. Affected target: Column "patients.status".',
     }));
     expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
 
     const allStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai-mini", scope: "all" },
+      payload: { modelId: "openai/gpt-4.1-mini", scope: "all" },
     });
     expect(allStart.statusCode).toBe(202);
     const allRun = await terminalRun(app, allStart.json().id);
@@ -327,7 +337,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     const missingStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai-mini", scope: "missing" },
+      payload: { modelId: "openai/gpt-4.1-mini", scope: "missing" },
     });
     expect(missingStart.statusCode).toBe(202);
     expect(await terminalRun(app, missingStart.json().id)).toMatchObject({