docs: research ThothII market pricing

.gitignore

@@ -35,7 +35,6 @@ config/ca-chain.pem
 
 # ThothII deployment configuration and secret values (keep only the README tracked)
 deploy/.env
-deploy/env/local.env
 deploy/compose.connector-secrets.local.yaml
 deploy/compose.psd-local.yaml
 deploy/workspaces/psd.yaml
@@ -46,7 +45,6 @@ deploy/secrets/*
 # Per-installation configuration generated by `tht setup` (examples stay tracked).
 deploy/*/thothii-installation.yaml
 deploy/*/operator.env
-deploy/*/generated/
 deploy/*/secrets/*
 !deploy/*/secrets/.gitkeep
 !deploy/*/secrets/*.example

AGENTS.md

@@ -83,8 +83,7 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
   `SseHub` fans them out over SSE to the browser. The separate PostgreSQL catalog stores database
   metadata and sequential AI description-generation runs. Description generation samples the DWH
   through read-only connectors and calls a short-lived Python LiteLLM helper; it does not use Pi or
-  expose a public CLI command. Sessions, metadata generation, and embedding resolve models from the
-  generated Installation Model Catalog; `thothii-installation.yaml` is its only authored source.
+  expose a public CLI command. App settings still live in `backend/data/settings.json`.
 
 - **Human-in-the-loop gate contract.** The model proposes; a human reviewer decides at gates
   via widgets (`reviewer_select` = single pick — a chosen option carrying a `decision` payload
@@ -100,12 +99,11 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
 - **`--json` output must be pristine** (only valid JSON on stdout) — used as a machine contract.
 - **UI strings are English; document *content* stays the workspace language** (Italian for
   `psd`) because it's the real data. Only chrome/labels are English.
-- **Workspace schema v4** defines workspace identity and optional Evidence only. PostgreSQL Metadata
-  Catalog owns database identity, binding, schema, descriptions, sensitivity, and relationships;
-  embedding/model facts come from the installation catalog. The legacy `harness/workspaces/*.yaml` runtime snapshots still use
-  absolute session/artifact/index paths; secrets stay in `harness/.env` (gitignored).
-- **Settings are global** (`backend/data/settings.json`: workspace/thinking). Provider/model choices
-  are ephemeral canonical catalog selections pinned into the session manifest.
+- **Workspaces** (`harness/workspaces/*.yaml`) set the DB target and **absolute**
+  `paths.sessions/artifacts/indexes` — for `psd` these point at a *separate, uncommitted* repo
+  (`tht-workspace-psd/`). Secrets live ONLY in `harness/.env` (gitignored).
+- **Settings are global** (`backend/data/settings.json`: workspace/provider/model/thinking);
+  the New-session form is question-only.
 - **Resume**: a resumable session re-enters at its last incomplete phase. The backend refuses
   resume with 409 when `finalized` or `archived`, and `PiProcessManager.spawnFor` must send
   `/riprendi-sessione <id>` (resume mode) vs `/nuova-domanda` (new) — sending the wrong prompt

CONTEXT.md

@@ -230,18 +230,10 @@ conversione degli a-capo e rimozione degli spazi esterni. Gli elementi contestua
 poi deduplicati e ordinati senza conversione delle maiuscole, mentre punteggiatura e
 spazi interni della domanda non vengono riscritti.
 
-**Reference Vector Collection** — La collezione Qdrant ricostruibile di un workspace che
-contiene Schema, relazioni ed Evidence. Possiede il vettore dense predefinito e il vettore
-sparse `bm25`; soltanto gli Evidence Fragment ricevono valori BM25. Il preprocessing può
-sostituirla o eliminarla integralmente.
-
-**Memory Vector Collection** — La collezione Qdrant persistente di un workspace che contiene
-`memory` e `solved_question`. Non è un output del preprocessing e non viene eliminata dal
-Preprocessing Clear.
-
-**Preprocessing Clear** — L'operazione amministrativa che elimina Reference Vector Collection,
-LSH, corpus e checkpoint derivati e rende il workspace non pronto. Conserva Memory Vector
-Collection, sessioni, Catalog Metadata e database sorgente; non offre history o rollback.
+**Additive BM25 upgrade** — L'estensione non distruttiva della collezione semantica di
+un workspace che conserva il vettore dense predefinito e aggiunge il solo vettore
+sparse `bm25`. Soltanto gli Evidence Fragment ricevono valori BM25; Schema e Memory
+mantengono invariati dati e ricerca dense.
 
 **Formula proposal** — Una formula individuata durante una sessione e conservata come
 artefatto della sessione. Non diventa Published Evidence finché non viene importata,
@@ -254,10 +246,9 @@ precedente o di un altro workspace.
 
 ## Configurazione dei modelli
 
-**Workspace Descriptor** — La dichiarazione versionata dell'identità del workspace e dello
-scope delle sue Evidence. Non contiene identità o configurazione del Workspace Database,
-Database Binding, fatti strutturali o metadati semantici: il Metadata Catalog associa il
-workspace al relativo database.
+**Workspace Descriptor** — La dichiarazione versionata dell'identità e dello scope del
+Workspace Database e delle Evidence di un workspace. Non definisce modelli, selezioni di
+modello o Database Binding specifiche di un'installazione.
 
 **Installation Model Catalog** — L'insieme dichiarativo, proprio di un'installazione, dei
 modelli disponibili, dei loro Model Usage e dei relativi default. È l'unica autorità per i
@@ -276,21 +267,6 @@ ridefinirne provider, endpoint o capacità.
 dell'Installation Model Catalog richiesta da uno specifico runtime. Può essere rigenerata
 integralmente dalla configurazione dell'installazione.
 
-## Distribuzione del prodotto
-
-**Customer-Hosted Installation** — Un'installazione eseguita interamente nel trust boundary
-controllato dall'organizzazione cliente, inclusi eventuali tenant cloud privati. Credenziali,
-domande, prompt, metadati e risultati non attraversano quel boundary.
-_Avoid_: on-premise deployment, self-managed deployment
-
-**Community Edition** — La distribuzione open source utilizzabile gratuitamente anche in
-produzione e capace di eseguire il workflow fondamentale completo.
-_Avoid_: free tier, trial edition
-
-**Enterprise Edition** — La distribuzione con licenza commerciale che aggiunge governance
-organizzativa, esercizio production-grade e industrializzazione alla Community Edition.
-_Avoid_: paid tier, pro edition
-
 ## Catalogo dei metadati
 
 **Workspace Database** — Il database che appartiene a un solo workspace e non può essere
@@ -311,9 +287,8 @@ client configurabile per API REST arbitrarie.
 nel catalogo autorevole. Rimane conservato per il recupero amministrativo, ma non può essere
 usato dal workflow finché non viene riassegnato a un workspace esistente.
 
-**Metadata Catalog** — L'autorità per l'associazione fra workspace e Workspace Database, la
-relativa Database Binding, i fatti strutturali osservati e i metadati semantici curati. Ogni
-uso downstream dei metadati del database deriva da questo catalogo.
+**Metadata Catalog** — Il contesto amministrativo che raccoglie e cura i metadati di un
+Workspace Database. Non definisce quali elementi partecipano al workflow SQL.
 
 **Database Profile** — L'insieme curato di scope, descrizioni e metadati semantici
 associato a un Workspace Database.
@@ -375,19 +350,15 @@ logicamente.
 Logical Relationship, con origine e stato espliciti. È l'interfaccia usata dall'amministrazione e
 dalla comprensione dello schema, non un ulteriore modello persistito.
 
-**Catalog Metadata Snapshot** — La proiezione immutabile e versionata della struttura catalogata,
-delle descrizioni pubblicabili e delle relazioni effettive attive di un Workspace Database che il
-core consuma. È derivata esclusivamente dal Metadata Catalog e non è un archivio autoritativo.
-
-**Schema Index** — La proiezione vettoriale ricostruibile dei metadati del Workspace Database nel
-Metadata Catalog. Il preprocessing la sostituisce integralmente e non è una fonte di verità.
+**Effective Relationship Snapshot** — La proiezione runtime immutabile delle relationship attive
+contenute nell'Effective Relationship Map. È derivata dal Metadata Catalog per una singola sessione
+e viene eliminata insieme alla relativa configurazione runtime.
 
 **Description** — Il testo curato e consolidato che descrive una Catalog Table o Catalog Column
-per gli usi downstream. Quando presente, prevale sulla relativa Generated Description.
+per gli usi downstream.
 
-**Generated Description** — Il testo modificabile prodotto dall'AI per una Catalog Table o Catalog
-Column. È pubblicabile per gli usi downstream quando manca una Description, anche senza essere
-prima consolidato, e rimane distinto dal commento osservato nel database.
+**Generated Description** — Una proposta modificabile sottoposta a revisione umana prima di
+essere consolidata come Description. Rimane distinta dal commento osservato nel database.
 _Avoid_: generated comment, source comment
 
 **Description Consolidation** — L'azione amministrativa esplicita che copia la Generated
@@ -433,17 +404,9 @@ Schema Synchronization.
 della Database Binding. Uno scope rimane consultabile ma è stale finché non viene sincronizzato
 con la binding corrente.
 
-**Metadata Content Revision** — La revisione monotona di tutto lo stato del Metadata Catalog che
-può modificare il comportamento del core. Ogni mutazione rilevante produce una nuova revisione
-nella stessa transazione che la rende durevole.
-
-**Preprocessing State** — Lo stato corrente `running`, `succeeded` o `failed` del preprocessing di
-un workspace, insieme all'identità dei suoi input. Il core può usare il workspace soltanto quando
-lo stato è `succeeded` e gli input coincidono ancora.
-
 **Catalog Metadata** — I campi mutabili che descrivono database, tabelle, colonne e relazioni,
 distinti dai fatti strutturali governati dalla sincronizzazione. Possono essere popolati dall'AI,
-o da una modifica amministrativa senza cambiare il database esterno.
+da un'importazione o da una modifica amministrativa senza cambiare il database esterno.
 
 **Model Completion Helper** — Il processo Python interno ed effimero che esegue una singola
 richiesta LiteLLM per conto del backend. Non è un servizio HTTP, non possiede il lifecycle della
@@ -457,12 +420,6 @@ diventa Catalog Metadata.
 **Sensitive Data Flag** — La classificazione binaria umana applicata a una Catalog Column. Può
 essere impostata liberamente dall'amministratore anche in contrasto con una valutazione automatica.
 
-**Sensitivity Reason** — La motivazione sanificata persistita insieme al Sensitive Data Flag
-quando l'amministratore salva una Sensitivity Review Draft. È Catalog Metadata della colonna, non
-history della run; viene rimossa quando il flag torna non-sensitive e può essere assente per una
-classificazione manuale priva di valutazione locale.
-_Avoid_: AI reasoning, source evidence
-
 **Local Sensitivity Assessment** — La valutazione locale, non autoritativa e priva di LLM di una
 Catalog Column, basata su metadati e contenuto sorgente, con esito `sensitive`, `non_sensitive`
 oppure `unknown`.
@@ -494,13 +451,12 @@ _Avoid_: Sensitive Data Suggestion Run, AI analysis
 
 **Sensitivity Review Draft** — La proposta transitoria che associa alle colonne selezionate una
 Local Sensitivity Assessment e le relative evidenze sanificate. Non modifica il Sensitive Data Flag
-né la Sensitivity Reason finché l'amministratore non salva le proprie decisioni e viene scartata al
-reload.
+finché l'amministratore non salva le proprie decisioni e viene scartata al reload.
 _Avoid_: automatic flag
 
 **Sensitivity Analysis Event** — Una riga testuale ordinata e sanificata che registra l'avvio,
-l'avanzamento per fase e batch, l'esito o l'errore di una Sensitivity Analysis Run senza conservare
-contenuti sorgente, output grezzi del detector o proposte per colonna.
+l'esito o l'errore di una Sensitivity Analysis Run senza conservare contenuti sorgente, output grezzi
+del detector o proposte per colonna.
 _Avoid_: Sensitive Data Suggestion Event
 
 **Introspection Capability** — Una categoria di struttura fisica che una Database Binding

PROJECT_STATE.md

@@ -1,17 +1,12 @@
 # ThothII — Project State
 
-Last updated: 2026-09-06.
+Last updated: 2026-08-31.
 
 This file is the short operational snapshot. Stable commands and the architecture mental model
 live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`,
 `docs/contracts/`, `docs/adr/`, and `docs/evidence.md`. Superseded plans and reports are
 available from Git history rather than duplicated in the working tree.
 
-The guarded server migration from a legacy checkout to the schema-v2 installation, Gitea source,
-Authentik, internal catalog/embedding services, and the PSD workspace repository is documented in
-`docs/operations/server-upgrade-gitea-workspace-v2.md`. Treat its operator gates and rollback
-requirements as mandatory; do not replace the running server stack in place.
-
 ## Current product shape
 
 ThothII is a human-in-the-loop datamart builder with three independently built layers:
@@ -51,56 +46,23 @@ The canonical authoring, validation, publication, materialization, and preproces
 documented in `docs/evidence.md`. The governing contracts are
 `docs/contracts/workspace-evidence-v3.md` and
 `docs/contracts/workspace-preprocessing-cli.md`.
-The incremental server procedure for the `260906-preprocessing-complete` release is
-`docs/operations/server-handoff-260906-preprocessing-complete.md`.
 
 ## Workspace preprocessing and configuration
 
 The native host CLI `tht` is the operator surface. Workspace preprocessing runs through:
 
 ```sh
-tht --installation /absolute/path/thothii-installation.yaml \
-  workspace preprocess run --workspace <workspace-id>
+tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
+tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
 ```
 
-This complete one-shot command uses the profile-gated `workspace-maintenance` service. Partial DWH,
-schema, Evidence, and vector mutation commands are retired.
+These commands use the profile-gated `workspace-maintenance` service. The former standalone
+preprocessing Compose fixtures are retired.
 
-The right Administration sidebar invokes that same operation for the selected workspace. It shows
-only current readiness or the latest bounded failure diagnostic; there is no preprocessing history.
-Known non-ready state disables **New session**, while backend admission remains authoritative.
-The same control exposes an inline-confirmed **Clear** action to remove replaceable reference
-vectors, LSH, corpus, and checkpoints while preserving the separate Memory collection. The host CLI
-equivalent is `workspace preprocess clear`.
-
-Each workspace now uses `<workspace>-reference` for Schema, relationships, and Evidence and
-`<workspace>-memory` for `memory` and `solved_question`. Clear and preprocessing own only the former.
-LSH ownership additionally binds the Catalog database ID and Metadata Content Revision, so derived
-values cannot be reused across database identities or Catalog revisions.
-
-Workspace descriptors use schema v4 and contain only workspace identity and optional Evidence.
-PostgreSQL Metadata Catalog owns database identity, binding, schema, descriptions, sensitivity, and
-relationships; model, provider, embedding, and vector-store configuration is installation-owned. For
-PSD, workspace content and runtime roots point to the
+Workspace descriptors use schema v3. For PSD, workspace content and runtime roots point to the
 separate uncommitted repository `/Users/mp/projects/tht-workspace-psd`. Secrets remain outside
 Git and are supplied only through installation-local protected files.
 
-## Installation Model Catalog
-
-`thothii-installation.yaml` schema version 2 is the only operator-authored source for session,
-metadata-generation, and embedding models. The host `tht` lifecycle validates `modelCatalog` and
-regenerates the backend catalog, Pi `models.json`/`settings.json`, and Compose override under the
-installation-local `generated/` directory. Those projections are replaceable runtime adapters:
-they are not edited, backed up, or treated as configuration.
-
-Session and metadata defaults use canonical `provider/model` IDs. Provider authentication declares
-one explicit mode (`secret_env`, `pi_auth`, or `none`); `secret_env` names a protected bundle key.
-The backend settings store now owns only the selected workspace and thinking level. Existing v1
-installations use the explicit catalog migration command; schema-v3 workspace descriptors are
-converted deterministically in their curator-owned repository before commit. Strict runtime loading
-does not silently infer or merge legacy sources. ADR 0013 and
-`docs/plans/2026-09-02-installation-model-catalog.md` record the decision and implementation.
-
 ## Database management
 
 The database, table, and authoritative physical-schema catalog slices are implemented. Database
@@ -118,23 +80,6 @@ column. The KPI strip reads installation-wide or selected-database aggregates fr
 description history, and sensitive-field review/history use the production APIs in right-side
 drawers rather than prototype fixtures; closing a history drawer does not stop its background run.
 
-Sensitive-field review is now driven by the versioned local `sensitivity-v4` policy, not by a
-catalog model. The backend reads selected source tables through read-only, database-specific
-adapters and makes every `sensitive | non_sensitive` draft decision in the TypeScript
-`SensitivityClassifier`. A single validated match protects the column. Tables up to 1,000 rows are
-fully scanned; larger tables use breadth-first 300, 1,000, and text-only 3,000-value targets, with a
-five-second limit per source query and no global request deadline. Source failures fail the run
-instead of yielding `unknown`; coverage remains visible separately from the proposal. Draft
-assessments remain transient until an administrator explicitly saves them. Optional GLiNER2
-evidence is CPU-only, offline, opt-in, and never replaces the deterministic decision point; see
-`docs/operations/sensitivity-analysis.md`. The earlier v1 PSD shadow comparison kept NER disabled by
-default; see `docs/reports/2026-09-02-psd-sensitivity-shadow.md`. The v2 comparison completed all
-2,275 columns: CPU NER added 18 sensitive proposals and increased warm runtime from 50.1 to 61.3
-seconds; see `docs/reports/2026-09-03-psd-progressive-sensitivity-shadow.md`.
-Version 4 excludes declared `bigint` primary-key columns and conventionally named `pk bigint`
-columns before source inspection, reporting both as non-informative structural identifiers while
-distinguishing declared constraints from inferred roles.
-
 Physical membership, source
 comments, column types/default/nullability/PK positions, and constraint-level ordered FK pairs are
 projections of the external schema. They cannot be created, renamed, or structurally edited by
@@ -181,29 +126,26 @@ SSH is not yet enabled for NL→SQL session runtime.
 
 The catalog runs in the internal `catalog-db` PostgreSQL service. Kysely migrations are an explicit
 one-shot `catalog-migrate` operation; `scripts/run-stack.sh` runs it before local startup. Runtime
-sessions consume an immutable Catalog JSON snapshot tied to the runtime-config lease. It contains
-the tables, columns, effective descriptions, sensitivity flags, and active relationships used by
-the harness; PostgreSQL is the exclusive runtime authority for database metadata. Authored
-workspace YAML remains limited to workspace identity and optional Evidence configuration. The
-accepted design is recorded in ADR 0016 and the contracts under `docs/contracts/`.
+sessions now consume the Catalog's active effective relationship map through an immutable JSON
+snapshot tied to the runtime-config lease. The harness uses that snapshot as its exclusive
+relationship source while retaining Git-pinned annotations for descriptive metadata; legacy
+runtimes without a snapshot keep the previous merge behavior. The accepted design is recorded in
+`docs/plans/2026-08-26-metadata-catalog-from-thothai.md`, the snapshot contract under
+`docs/contracts/`, and ADRs 0001–0012.
 
 Semantic aliases, value descriptions, synonyms, and concepts remain deferred to their dedicated
 slices.
 
 AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to
-`false`, including for newly synchronized columns. An administrator may request a local sensitivity
-analysis for one selected database, selected tables, or selected columns. One deterministic
-TypeScript classifier combines metadata, bounded source-content rules, and optional CPU-only NER;
-no generative model decides the result. Its `sensitive` or `non_sensitive` assessments remain an
-unsaved draft until the human reviews and saves any chosen flag changes, including a downgrade to
-non-sensitive. Coverage is reported separately; interrupted history may count unprocessed columns.
-Each started analysis records a separate Sensitivity Analysis Run with aggregate counters and safe
-ordered events. The progress drawer opens before the synchronous request completes, polls the run,
-and displays sanitized source-scan and local-NER phase/batch activity while classification is in
-progress. This operational history never stores per-column assessments, source values,
-matched spans, prompts, or free-form diagnostics. Saving a sensitive decision persists a sanitized
-Sensitivity Reason as column Catalog Metadata alongside the human-owned flag; clearing the flag
-clears that reason. Reloading still discards an unsaved review draft.
+`false`, including for newly synchronized columns. An administrator may request an AI proposal based
+only on structural metadata for one selected database, selected tables, or selected columns. The
+backend divides large scopes into deterministic model requests of at most ten columns, also bounded
+by helper message size, and combines their results, but the proposal remains an unsaved draft until
+the human reviews and saves it.
+Each started suggestion attempt records a separate Sensitive Data Suggestion Run with aggregate
+counters and safe ordered events. This operational history never stores per-column proposals,
+prompts, raw model output, or provider diagnostics; reloading still discards an unsaved review
+draft.
 For unprotected columns, up to five source rows and five representative non-null values may be sent
 transiently to the configured model provider. Protected columns are omitted from source reads and
 replaced in the prompt by deterministic plausible values derived only from their metadata. Existing
@@ -222,8 +164,7 @@ available only when no local start, worker, or helper is live. Runs remain inspe
 live SSE log with ordered polling fallback; there is no automatic resume or user-facing generation
 CLI. ADRs 0009–0010 record the runtime and source-sampling decisions.
 
-The Installation Model Catalog accepts the protected `DEEPSEEK_API_KEY` and `ZAI_API_KEY`
-references for metadata-generation providers.
+Metadata-generation setup accepts the protected `DEEPSEEK_API_KEY` and `ZAI_API_KEY` references.
 It also accepts a model with no secret reference only when its OpenAI-compatible endpoint is
 explicit; this covers the VPN-only AritmoLab Qwen 3.6 server without creating a fake operator
 credential. The Python client supplies only its fixed non-secret compatibility placeholder.

README.md

@@ -106,20 +106,15 @@ a remote user's partial list. The isolated deployment exercise is
 `./scripts/verify-workspace-install-docs.sh --profile local` or `--profile server`.
 
 <!-- workspace-descriptor-contract:start -->
-Schema v4 is the only accepted workspace descriptor. It contains workspace identity and optional
-Evidence configuration only; PostgreSQL Metadata Catalog owns every database fact and binding.
-Schema v1, v2, and v3 descriptors are rejected before activation. Candidate snapshot validation
-therefore makes activation or a pull fail atomically while the prior valid snapshot remains active.
-Each workspace owns separate Qdrant `reference` and `memory` collections: Schema, relationships, and
-Evidence are replaceable reference data; Memory and solved questions have a persistent lifecycle.
+Schema v3 is the only accepted workspace descriptor. Schema v1 and v2 workspace descriptors are
+rejected before activation. Candidate snapshot validation therefore makes activation or a pull fail
+atomically while the prior valid snapshot remains active. There is no in-product migrator or
+automatic conversion. A repository must already contain reviewed v3 descriptors. One workspace
+owns one Qdrant collection;
+schema, Evidence, and Memory records share that collection and stay separated by indexed payload
+`kind`.
 <!-- workspace-descriptor-contract:end -->
 
-<!-- non-workspace-migration:start -->
-Create a clean v4 descriptor containing only `workspace` and optional `evidence`. Do not copy the
-legacy database, diagnostics, `llm_policy`, or `semantic_index` blocks; configure the database in
-Database Management.
-<!-- non-workspace-migration:end -->
-
 For NL→SQL runtime sessions, connector `ssh_tunnel` bindings remain diagnostic-only: their bounded
 probe cleans up the loopback forward and returns `workspace_not_activatable`; session creation is
 rejected before persistence. Database management is a separate boundary and supports a strict
@@ -181,10 +176,10 @@ secret files, upstream-auth checks, and a fail-closed `503` assertion for its de
 unavailable disposable session endpoint. No real provider, database credential, or repository
 secret is required.
 
-For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular Pi agent
-mount targets atomically before Compose. The auth target receives the protected credential bind;
-the model and settings targets receive generated read-only projections. The server smoke starts
-from an empty Pi-state root and applies this same preflight. Deterministic fixture tests render
+For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular
+`agent/auth.json`, `agent/models.json`, and `agent/settings.json` mount targets atomically before
+Compose. The server smoke starts from an empty Pi-state root and applies this same preflight; the
+real protected/tracked sources remain separate read-only mounts. Deterministic fixture tests render
 both profiles, verify that bindings stay on `core`, check mount readability, and run the production
 workspace resolver. Wrong-service, wrong-value, and broken-secret-mount mutations must fail.
 
@@ -194,7 +189,7 @@ an independent 32-minute outer timeout and does not retry a failed command.
 Current release status (2026-08-05): clean-root render/setup and the production runtime-binding
 resolver contracts are green. The server fixture supplies all four private trusted claims,
 including exact non-admin value `0`, and a focused test proves nginx normalization produces the
-accepted non-admin backend principal. Canonical schema-v4 registry descriptors now pass through
+accepted non-admin backend principal. Canonical schema-v3 registry descriptors now pass through
 one backend-owned, secret-safe runtime handoff for inventory and session execution; canonical
 identity and durable session/artifact/index roots are retained. The fresh update-only smoke passed
 bad-candidate mutation, automatic `rolled_back` compensation, exact prior-image restoration,
@@ -225,23 +220,15 @@ job remains deterministic and does not claim Docker startup.
 
 ## Workspace preprocessing and S3 Evidence
 
-For an interactive run, select the workspace, expand **Administration** in the right sidebar, and
-use its **Preprocessing** control. The control explains any unmet prerequisite and exposes only the
-latest safe failure diagnostic. For unattended operation, use the native host CLI and installation
-descriptor:
+Run preprocessing through the native host CLI and the installation descriptor:
 
 ```sh
-tht --installation /absolute/path/thothii-installation.yaml \
-  workspace preprocess run --workspace <workspace-id>
-
-tht --installation /absolute/path/thothii-installation.yaml \
-  workspace preprocess clear --workspace <workspace-id>
+tht --installation /absolute/path/thothii-installation.yaml workspace preprocess evidence
+tht --installation /absolute/path/thothii-installation.yaml workspace preprocess dwh
 ```
 
-The one-shot command starts the profile-gated `workspace-maintenance` service, reads database
-metadata from PostgreSQL, and rebuilds LSH plus schema/Evidence vectors. The clear command removes
-those derived artifacts while preserving the separate Memory collection. The core remains unavailable
-until preprocessing completes. See [Evidence](docs/evidence.md) and the
+The CLI starts the profile-gated `workspace-maintenance` service and enforces the workspace,
+secret, Qdrant, and embedding contracts. See [Evidence](docs/evidence.md) and the
 [workspace preprocessing CLI contract](docs/contracts/workspace-preprocessing-cli.md).
 
 S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance.
@@ -282,7 +269,7 @@ Compose project name by passing `--confirm-project`:
 The restore script stops `qdrant`, validates the exact labeled target, stages the current volume
 contents for rollback, extracts the requested archive into the volume, and then returns the
 service to its prior running state. It restores semantic storage only. Before reopening write
-traffic, the workspace registry must already be at a reviewed v4 descriptor revision compatible
+traffic, the workspace registry must already be at a reviewed v3 descriptor revision compatible
 with the restored collection; then run backend health checks and a known retrieval query. The
 helper does not restore descriptors, rename collections, or reconcile an incompatible collection
 contract.
@@ -308,12 +295,14 @@ Copy `deploy/secrets/thothii.secrets.example` to a protected host file, include
 keys, and set its absolute path as `THT_SECRETS_FILE` in the operator env. Keep Pi's native
 provider auth in the separate protected file named by `PI_AUTH_FILE`.
 
-Interactive sessions, Description Generation, and embedding share the protected installation
-descriptor's `modelCatalog`. Set `THT_INSTALLATION_CONFIG_SOURCE` to that exact host file; `tht`
-validates it and generates the runtime catalog, Pi adapters, and Compose override before startup.
-Each authenticated provider stores only an audited `apiKeyEnv` reference; the referenced value stays
-in the secret bundle. A provider may use `authentication.mode: none` only with an explicit keyless
-endpoint. The browser receives only eligible model IDs, labels, and the catalog default.
+Description Generation is configured independently in the protected installation descriptor under
+`metadataGeneration`. Set `THT_INSTALLATION_CONFIG_SOURCE` to that exact host file; Compose mounts
+it read-only into `core` and supplies the fixed runtime `THT_INSTALLATION_CONFIG_FILE` path. Each
+keyed model stores only an audited `apiKeyEnv` reference. The referenced value stays in the secret
+bundle; a model may omit `apiKeyEnv` only when it declares an explicit endpoint that accepts
+unauthenticated requests. The browser receives only model IDs, labels, and the configured default.
+Configuration changes take effect after restart and do not use Pi settings or workspace
+`llm_policy`.
 
 Before enabling Description Generation, approve the selected model provider for bounded source-data
 disclosure. Every catalog column has a **Sensitive** flag that defaults to `false`. Administrators can
@@ -344,11 +333,22 @@ the host/secret-manager materialization and add a reviewed Compose override that
 does not create that mount. The frontend remains on loopback; the authenticated host proxy is the
 only public listener.
 
-For each Pi spawn, the backend resolves the selected canonical provider/model in the runtime catalog,
-reads exactly that provider's declared `apiKeyEnv` value from the bundle, and exposes only that key
-to the child. Ambient provider credentials and secret-bundle paths are scrubbed. Providers needing a
-compound credential bundle remain unsupported until the catalog gains an explicit generic contract
-for them.
+Set the selected model provider in application settings (or `PI_PROVIDER`). For each Pi spawn the
+backend validates and reads `THT_MODEL_API_KEY` from the bundle, then exposes its value only as the provider's
+recognized child variable (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or
+`ZAI_API_KEY`). Neither the generic file path nor deprecated `PI_PROVIDER_API_KEY` is inherited by
+Pi. Local providers such as Ollama require no model key.
+
+`THT_MODEL_API_KEY` supports Pi providers whose authentication is exactly one key:
+`ant-ling`, `anthropic`, `cerebras`, `deepseek`, `fireworks`, `github-copilot`, `google`
+(including the `gemini` alias), `google-vertex` when using its API-key mode, `groq`,
+`huggingface`, `kimi-coding`, `minimax`, `minimax-cn`, `mistral`, `moonshotai`,
+`moonshotai-cn`, `nvidia`, `openai`, `opencode`, `opencode-go`, `openrouter`, `together`,
+`vercel-ai-gateway`, `xai`, the four `xiaomi*` providers, `zai`, and `zai-coding-cn`.
+Compound providers are deliberately unsupported: `amazon-bedrock`, `azure-openai-responses`,
+`cloudflare-workers-ai`, and `cloudflare-ai-gateway` require multiple credential/configuration
+values. Selecting one fails before Pi starts; ambient AWS, Azure, and Cloudflare credentials are
+still scrubbed. Supporting them requires a future dedicated provider-specific configuration.
 
 ## User-owned session server cutover
 
@@ -357,8 +357,6 @@ The server profile stores sessions and per-user preferences directly in PostgreS
 dual write. Use [`deploy/compose.session-server.yaml.example`](deploy/compose.session-server.yaml.example)
 with the canonical base+server files and set `THT_SERVER_WORKSPACE_CONFIG` to an absolute,
 protected copy of [`deploy/workspaces/server-sessions.yaml.example`](deploy/workspaces/server-sessions.yaml.example).
-That file is an installation runtime template, not an authored workspace descriptor; database
-bindings are injected from the PostgreSQL Metadata Catalog for each runtime lease.
 
 The runtime login needs membership in the no-login database role `thoth_sessions_runtime` only.
 The distinct, one-shot migrator login needs migration authority and uses

backend/package-lock.json

@@ -12,17 +12,14 @@
         "@types/pg": "^8.20.3",
         "fastify": "^5.0.0",
         "kysely": "^0.29.5",
-        "libphonenumber-js": "1.13.12",
         "openid-client": "6.8.5",
         "pg": "^8.22.0",
-        "validator": "13.15.35",
         "yaml": "^2.9.0",
         "zod": "^4.4.3"
       },
       "devDependencies": {
         "@testcontainers/postgresql": "^12.1.0",
         "@types/node": "24.13.3",
-        "@types/validator": "13.15.10",
         "tsx": "^4.19.0",
         "typescript": "^5.6.0",
         "vitest": "^2.1.0"
@@ -1332,13 +1329,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/@types/validator": {
-      "version": "13.15.10",
-      "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.10.tgz",
-      "integrity": "sha512-T8L6i7wCuyoK8A/ZeLYt1+q0ty3Zb9+qbSSvrIVitzT3YjZqkTZ40IbRsPanlB4h1QB3JVL1SYCdR6ngtFYcuA==",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/@vitest/expect": {
       "version": "2.1.9",
       "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz",
@@ -2468,9 +2458,9 @@
       }
     },
     "node_modules/fast-uri": {
-      "version": "3.1.7",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz",
-      "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==",
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
+      "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
       "funding": [
         {
           "type": "github",
@@ -2484,9 +2474,9 @@
       "license": "BSD-3-Clause"
     },
     "node_modules/fastify": {
-      "version": "5.12.3",
-      "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.12.3.tgz",
-      "integrity": "sha512-reZ8wce5VNCcufIt9AVtzZa3L4u1j8esikn7OEgHWLVpRpL5R7Y2+Xzj70OUkv5zDfzUAxXZT6cu4Rt0zr3EKA==",
+      "version": "5.8.5",
+      "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.8.5.tgz",
+      "integrity": "sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==",
       "funding": [
         {
           "type": "github",
@@ -2505,11 +2495,11 @@
         "@fastify/proxy-addr": "^5.0.0",
         "abstract-logging": "^2.0.1",
         "avvio": "^9.0.0",
-        "fast-json-stringify": "^7.0.0",
-        "find-my-way": "^9.6.0",
+        "fast-json-stringify": "^6.0.0",
+        "find-my-way": "^9.0.0",
         "light-my-request": "^6.0.0",
         "pino": "^9.14.0 || ^10.1.0",
-        "process-warning": "^5.1.0",
+        "process-warning": "^5.0.0",
         "rfdc": "^1.3.1",
         "secure-json-parse": "^4.0.0",
         "semver": "^7.6.0",
@@ -2532,46 +2522,6 @@
       ],
       "license": "MIT"
     },
-    "node_modules/fastify/node_modules/fast-json-stringify": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-7.0.1.tgz",
-      "integrity": "sha512-eRSayARSbbwlBjpP4vnTTIRD5QPcIrmihPxDeN1DtKnHPg66UuJLx+8hlK1kaFdjvzyQ/dzALoi4vwAQ+T+iZA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/fastify"
-        },
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/fastify"
-        }
-      ],
-      "license": "MIT",
-      "dependencies": {
-        "@fastify/merge-json-schemas": "^0.2.0",
-        "ajv": "^8.12.0",
-        "ajv-formats": "^3.0.1",
-        "fast-uri": "^4.0.0",
-        "json-schema-ref-resolver": "^3.0.0",
-        "rfdc": "^1.2.0"
-      }
-    },
-    "node_modules/fastify/node_modules/fast-uri": {
-      "version": "4.1.4",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.4.tgz",
-      "integrity": "sha512-dODXrIxlS9JSdgAnhIUKOosKV1oMtU2VtVw87QRaHzyl5jxO290Ii5tEZfCfzfWNHi3jKWwBSdQj0qIyshdZdQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/fastify"
-        },
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/fastify"
-        }
-      ],
-      "license": "BSD-3-Clause"
-    },
     "node_modules/fastq": {
       "version": "1.20.1",
       "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
@@ -2874,12 +2824,6 @@
         "safe-buffer": "~5.1.0"
       }
     },
-    "node_modules/libphonenumber-js": {
-      "version": "1.13.12",
-      "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.13.12.tgz",
-      "integrity": "sha512-uLVeV1c9OTk6qkdqnj+mpMD+ZdnZ0szVyWu58HwMmpwkHA1gCEkyjd3veZQXDnuw9KEwSRjcc9B1pS9XKIN1fA==",
-      "license": "MIT"
-    },
     "node_modules/light-my-request": {
       "version": "6.6.0",
       "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz",
@@ -3027,9 +2971,9 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "3.3.18",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz",
-      "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==",
+      "version": "3.3.15",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz",
+      "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==",
       "dev": true,
       "funding": [
         {
@@ -3281,9 +3225,9 @@
       "license": "MIT"
     },
     "node_modules/postcss": {
-      "version": "8.5.28",
-      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz",
-      "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==",
+      "version": "8.5.15",
+      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
+      "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
       "dev": true,
       "funding": [
         {
@@ -3301,7 +3245,7 @@
       ],
       "license": "MIT",
       "dependencies": {
-        "nanoid": "^3.3.18",
+        "nanoid": "^3.3.12",
         "picocolors": "^1.1.1",
         "source-map-js": "^1.2.1"
       },
@@ -3366,9 +3310,9 @@
       "license": "MIT"
     },
     "node_modules/process-warning": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz",
-      "integrity": "sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz",
+      "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==",
       "funding": [
         {
           "type": "github",
@@ -4177,15 +4121,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/validator": {
-      "version": "13.15.35",
-      "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.35.tgz",
-      "integrity": "sha512-TQ5pAGhd5whStmqWvYF4OjQROlmv9SMFVt37qoCBdqRffuuklWYQlCNnEs2ZaIBD1kZRNnikiZOS1eqgkar0iw==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 0.10"
-      }
-    },
     "node_modules/vite": {
       "version": "5.4.21",
       "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz",

backend/package.json

@@ -7,11 +7,9 @@
     "prebuild": "node scripts/clean-dist.mjs",
     "build": "tsc -p tsconfig.json",
     "catalog:migrate": "node dist/catalog/migrate.js",
-    "sensitivity:shadow": "node dist/catalog/sensitivity-shadow.js",
     "test": "vitest run",
     "start": "node dist/server.js",
-    "test:schema-v4-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs",
-    "test:schema-v3-verifier": "npm run test:schema-v4-verifier"
+    "test:schema-v3-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs"
   },
   "dependencies": {
     "@fastify/cookie": "11.1.2",
@@ -20,17 +18,14 @@
     "@types/pg": "^8.20.3",
     "fastify": "^5.0.0",
     "kysely": "^0.29.5",
-    "libphonenumber-js": "1.13.12",
     "openid-client": "6.8.5",
     "pg": "^8.22.0",
-    "validator": "13.15.35",
     "yaml": "^2.9.0",
     "zod": "^4.4.3"
   },
   "devDependencies": {
     "@testcontainers/postgresql": "^12.1.0",
     "@types/node": "24.13.3",
-    "@types/validator": "13.15.10",
     "tsx": "^4.19.0",
     "typescript": "^5.6.0",
     "vitest": "^2.1.0"

backend/python/sensitivity-ner-model-sha256.txt

@@ -1,8 +0,0 @@
-34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930  ./.gitattributes
-4d9344c58a2a2ea4bb4ff4f7c611a853cf413205fc10d0cace564eba06f73828  ./README.md
-180f0a10d1d5ed5ce3318db0bcb0b1b7780d79a52f0a8fc3acbd27f74536d0e4  ./THOTHII_MODEL_REVISION
-164f17362bcf9d114067d3465e7374bfdd79ce6b605acb745de5a49dabb9595c  ./config.json
-f27dd63cc43a248d2566f0b6ad7a115db353676ce0561dcbca45bac766464c1a  ./encoder_config/config.json
-0280f6f39f6012da50b6640bad438d9b7e763a1b0102094115d1b710c4dd79b6  ./model.safetensors
-f6df10ec83bea993035b2dd7c39345a3d4fcf23421c2adb6cb4ffc1e6d1bc4b5  ./tokenizer.json
-233beed1f1095cccfc7907cde31a8d90a0c6aa4fdfaf6493f8e55fd162e81ae6  ./tokenizer_config.json

backend/python/sensitivity-ner-requirements.txt

@@ -1,34 +0,0 @@
-# Optional offline CPU pack. Fully version-locked in its own venv; not part of the base image.
---extra-index-url https://download.pytorch.org/whl/cpu
-accelerate==1.14.0
-annotated-types==0.8.0
-certifi==2026.7.22
-charset-normalizer==3.5.1
-filelock==3.32.5
-fsspec==2026.7.0
-gliner2[local]==2.0.0
-hf-xet==1.6.0
-huggingface-hub==0.36.2
-idna==3.19
-Jinja2==3.1.6
-MarkupSafe==3.0.3
-mpmath==1.3.0
-networkx==3.6.1
-numpy==2.5.2
-packaging==26.3
-peft==0.20.0
-psutil==7.2.2
-pydantic==2.13.5
-pydantic-core==2.46.5
-PyYAML==6.0.3
-regex==2026.9.3
-requests==2.34.2
-safetensors==0.8.0
-sympy==1.14.0
-tokenizers==0.22.2
-torch==2.14.0+cpu
-tqdm==4.70.0
-transformers==4.57.6
-typing-extensions==4.16.0
-typing-inspection==0.4.4
-urllib3==2.7.0

backend/python/sensitivity_ner_worker.py

@@ -1,301 +0,0 @@
-"""Offline, CPU-only JSONL worker for optional sensitivity NER evidence."""
-
-from __future__ import annotations
-
-import argparse
-import contextlib
-import ctypes
-import errno
-import hashlib
-import json
-import os
-import socket
-import sys
-import tempfile
-from pathlib import Path
-from typing import Any
-
-
-PII_LABELS = [
-    "person",
-    "full_name",
-    "first_name",
-    "middle_name",
-    "last_name",
-    "date_of_birth",
-    "email",
-    "phone_number",
-    "address",
-    "street_address",
-    "city",
-    "state_or_region",
-    "postal_code",
-    "country",
-    "government_id",
-    "national_id_number",
-    "passport_number",
-    "drivers_license_number",
-    "license_number",
-    "tax_id",
-    "tax_number",
-    "bank_account",
-    "account_number",
-    "routing_number",
-    "iban",
-    "payment_card",
-    "card_number",
-    "card_expiry",
-    "card_cvv",
-    "username",
-    "ip_address",
-    "account_id",
-    "sensitive_account_id",
-    "password",
-    "secret",
-    "api_key",
-    "access_token",
-    "recovery_code",
-    "sensitive_date",
-    "document_date",
-    "expiration_date",
-    "transaction_date",
-]
-
-_MODEL_COMPAT_DIRECTORY: tempfile.TemporaryDirectory[str] | None = None
-_EXPECTED_MODEL_REVISION = "c153999da5f4c509df4322b0c6a1baf3d2c284d7"
-
-
-def _arguments() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument("--model", required=True)
-    parser.add_argument("--threads", type=int, default=2)
-    return parser.parse_args()
-
-
-def _disable_network() -> None:
-    libc = ctypes.CDLL(None, use_errno=True)
-    libc.prctl.argtypes = [
-        ctypes.c_int,
-        ctypes.c_ulong,
-        ctypes.c_ulong,
-        ctypes.c_ulong,
-        ctypes.c_ulong,
-    ]
-    libc.prctl.restype = ctypes.c_int
-    if libc.prctl(38, 1, 0, 0, 0) != 0:  # PR_SET_NO_NEW_PRIVS
-        raise RuntimeError("cannot enable no-new-privileges for network isolation")
-
-    try:
-        seccomp = ctypes.CDLL("libseccomp.so.2", use_errno=True)
-    except OSError as error:
-        raise RuntimeError("libseccomp is required for network isolation") from error
-    seccomp.seccomp_init.argtypes = [ctypes.c_uint32]
-    seccomp.seccomp_init.restype = ctypes.c_void_p
-    seccomp.seccomp_syscall_resolve_name.argtypes = [ctypes.c_char_p]
-    seccomp.seccomp_syscall_resolve_name.restype = ctypes.c_int
-    seccomp.seccomp_rule_add.argtypes = [
-        ctypes.c_void_p,
-        ctypes.c_uint32,
-        ctypes.c_int,
-        ctypes.c_uint,
-    ]
-    seccomp.seccomp_rule_add.restype = ctypes.c_int
-    seccomp.seccomp_load.argtypes = [ctypes.c_void_p]
-    seccomp.seccomp_load.restype = ctypes.c_int
-    seccomp.seccomp_release.argtypes = [ctypes.c_void_p]
-    seccomp.seccomp_release.restype = None
-
-    allow = 0x7FFF0000  # SCMP_ACT_ALLOW
-    deny = 0x00050000 | errno.EPERM  # SCMP_ACT_ERRNO(EPERM)
-    filter_context = seccomp.seccomp_init(allow)
-    if not filter_context:
-        raise RuntimeError("cannot initialize network syscall filter")
-    try:
-        for syscall in (
-            "socket",
-            "connect",
-            "sendto",
-            "sendmsg",
-            "sendmmsg",
-            "bind",
-            "listen",
-            "accept",
-            "accept4",
-        ):
-            syscall_number = seccomp.seccomp_syscall_resolve_name(syscall.encode("ascii"))
-            if syscall_number < 0:
-                raise RuntimeError(f"cannot resolve network syscall: {syscall}")
-            if seccomp.seccomp_rule_add(filter_context, deny, syscall_number, 0) != 0:
-                raise RuntimeError(f"cannot block network syscall: {syscall}")
-        if seccomp.seccomp_load(filter_context) != 0:
-            raise RuntimeError("cannot activate network syscall filter")
-    finally:
-        seccomp.seccomp_release(filter_context)
-
-    def blocked(*_args: Any, **_kwargs: Any) -> Any:
-        raise PermissionError(errno.EPERM, "network disabled")
-
-    socket.socket = blocked  # type: ignore[assignment]
-    socket.create_connection = blocked  # type: ignore[assignment]
-
-
-def _verify_model(path: Path) -> None:
-    revision_path = path / "THOTHII_MODEL_REVISION"
-    try:
-        revision = revision_path.read_text(encoding="utf-8").strip()
-    except OSError as error:
-        raise RuntimeError("model revision marker is unavailable") from error
-    if revision != _EXPECTED_MODEL_REVISION:
-        raise RuntimeError("model revision is not approved")
-
-    manifest_path = Path(__file__).with_name("sensitivity-ner-model-sha256.txt")
-    try:
-        manifest = manifest_path.read_text(encoding="utf-8").splitlines()
-    except OSError as error:
-        raise RuntimeError("model checksum manifest is unavailable") from error
-    for line in manifest:
-        checksum, separator, relative_name = line.partition("  ")
-        if not separator or len(checksum) != 64 or not relative_name.startswith("./"):
-            raise RuntimeError("model checksum manifest is invalid")
-        relative_path = Path(relative_name[2:])
-        if relative_path.is_absolute() or ".." in relative_path.parts:
-            raise RuntimeError("model checksum path is invalid")
-        model_file = path / relative_path
-        if not model_file.is_file() or model_file.is_symlink():
-            raise RuntimeError("approved model file is unavailable")
-        digest = hashlib.sha256()
-        with model_file.open("rb") as stream:
-            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
-                digest.update(chunk)
-        if digest.hexdigest() != checksum:
-            raise RuntimeError("approved model checksum does not match")
-
-
-def _transformers4_model_path(path: Path) -> Path:
-    """Adapt tokenizer metadata emitted by Transformers 5 without changing pinned weights.
-
-    GLiNER2 2.0.0 officially requires Transformers <5, while current Fastino checkpoints were
-    saved by Transformers 5.8.0.  Transformers 4 calls the same list
-    ``additional_special_tokens``; Transformers 5 renamed it to ``extra_special_tokens`` and
-    changed its type.  Keep the downloaded model immutable and create a temporary symlink view
-    containing only the compatibility metadata needed by the supported GLiNER2 dependency set.
-    """
-
-    tokenizer_path = path / "tokenizer_config.json"
-    try:
-        tokenizer = json.loads(tokenizer_path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError) as error:
-        raise RuntimeError("invalid tokenizer configuration") from error
-    extra_tokens = tokenizer.get("extra_special_tokens")
-    if extra_tokens is None:
-        return path
-    if not isinstance(extra_tokens, list) or not all(isinstance(token, str) for token in extra_tokens):
-        raise RuntimeError("unsupported extra_special_tokens configuration")
-    if "additional_special_tokens" in tokenizer:
-        raise RuntimeError("ambiguous special-token configuration")
-
-    global _MODEL_COMPAT_DIRECTORY
-    _MODEL_COMPAT_DIRECTORY = tempfile.TemporaryDirectory(prefix="thothii-ner-model-")
-    compatible_path = Path(_MODEL_COMPAT_DIRECTORY.name)
-    for child in path.iterdir():
-        if child.name == tokenizer_path.name:
-            continue
-        (compatible_path / child.name).symlink_to(child, target_is_directory=child.is_dir())
-    tokenizer["additional_special_tokens"] = tokenizer.pop("extra_special_tokens")
-    (compatible_path / tokenizer_path.name).write_text(
-        json.dumps(tokenizer, ensure_ascii=False, indent=2) + "\n",
-        encoding="utf-8",
-    )
-    return compatible_path
-
-
-def _load_model(model_path: str, threads: int) -> Any:
-    path = Path(model_path).resolve(strict=True)
-    if not path.is_dir():
-        raise RuntimeError("model path must be a local directory")
-    _verify_model(path)
-    os.environ["CUDA_VISIBLE_DEVICES"] = ""
-    os.environ["HIP_VISIBLE_DEVICES"] = ""
-    os.environ["HF_HUB_OFFLINE"] = "1"
-    os.environ["TRANSFORMERS_OFFLINE"] = "1"
-    import torch
-    from gliner2 import AutoExtractor
-
-    torch.set_num_threads(max(1, min(threads, 8)))
-    torch.set_num_interop_threads(1)
-    compatible_path = _transformers4_model_path(path)
-    with contextlib.redirect_stdout(sys.stderr):
-        model = AutoExtractor.from_pretrained(str(compatible_path), map_location="cpu")
-    _disable_network()
-    return model
-
-
-def _request(value: Any) -> tuple[str, list[dict[str, str]]]:
-    if not isinstance(value, dict) or not isinstance(value.get("id"), str):
-        raise ValueError("invalid request")
-    candidates = value.get("candidates")
-    if not isinstance(candidates, list) or not 1 <= len(candidates) <= 128:
-        raise ValueError("invalid candidates")
-    parsed: list[dict[str, str]] = []
-    for candidate in candidates:
-        if not isinstance(candidate, dict):
-            raise ValueError("invalid candidate")
-        column_id = candidate.get("columnId")
-        text = candidate.get("text")
-        if not isinstance(column_id, str) or not isinstance(text, str) or not 1 <= len(text) <= 500:
-            raise ValueError("invalid candidate")
-        parsed.append({"columnId": column_id, "text": text})
-    return value["id"], parsed
-
-
-def _detect(model: Any, candidates: list[dict[str, str]]) -> list[dict[str, Any]]:
-    evidence: list[dict[str, Any]] = []
-    for candidate in candidates:
-        result = model.extract_entities(
-            candidate["text"],
-            PII_LABELS,
-            threshold=0.5,
-            include_confidence=True,
-        )
-        entities = result.get("entities", {}) if isinstance(result, dict) else {}
-        best: tuple[str, float] | None = None
-        if isinstance(entities, dict):
-            for label, matches in entities.items():
-                if label not in PII_LABELS or not isinstance(matches, list):
-                    continue
-                for match in matches:
-                    if not isinstance(match, dict):
-                        continue
-                    confidence = match.get("confidence")
-                    if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
-                        continue
-                    if best is None or confidence > best[1]:
-                        best = (label, float(confidence))
-        if best is not None:
-            evidence.append(
-                {
-                    "columnId": candidate["columnId"],
-                    "label": best[0],
-                    "confidence": best[1],
-                }
-            )
-    return evidence
-
-
-def main() -> int:
-    args = _arguments()
-    model = _load_model(args.model, args.threads)
-    print(json.dumps({"ready": True}, separators=(",", ":")), flush=True)
-    for line in sys.stdin:
-        request_id = "invalid"
-        try:
-            request_id, candidates = _request(json.loads(line))
-            response = {"id": request_id, "ok": True, "evidence": _detect(model, candidates)}
-        except Exception:
-            response = {"id": request_id, "ok": False, "error": "detection_failed"}
-        print(json.dumps(response, separators=(",", ":")), flush=True)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

backend/scripts/p1-acceptance.mjs

@@ -1195,8 +1195,13 @@ export async function executeChecks({ checks, failAt, recorder } = {}) {
 
 function baseWorkspace(id, evidenceSource) {
   return {
-    workspace: { schema_version: 4, id, name: `P1 ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P1 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
   };
 }

backend/scripts/p1-manual-acceptance.mjs

@@ -109,7 +109,7 @@ async function validateDistFiles(repo,files){const dist=join(repo,"backend","dis
 
 export async function readManualOwnership({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),root=fixedManualRoot(repo);noSymlinkExisting(repo,root);let rootEntry,ownershipEntry;try{rootEntry=await lstat(root);ownershipEntry=await lstat(join(root,"ownership.json"));}catch{throw new Error("manual ownership is missing");}if(!rootEntry.isDirectory()||rootEntry.isSymbolicLink()||await realpath(root)!==root||!ownershipEntry.isFile()||ownershipEntry.isSymbolicLink())throw new Error("manual ownership is unsafe");let value;try{value=JSON.parse(await readFile(join(root,"ownership.json"),"utf8"));}catch{throw new Error("manual ownership is malformed");}const baseValid=value.schemaVersion===1&&value.kind==="p1-manual-acceptance"&&HEX64.test(value.nonce??"")&&value.repositoryRoot===repo&&value.root===root&&value.status==="PENDING"&&["PREPARING","READY"].includes(value.stage)&&value.listener?.host===HOST&&value.listener?.port===PORT&&value.listener?.state==="stopped"&&typeof value.createdAt==="string"&&validEntrypoint(value.entrypoint,repo)&&validDistManifest(value.distManifest,root)&&JSON.stringify(value.resources)===JSON.stringify([root,{kind:"fastify",host:HOST,port:PORT}]);const readyLog=value.backendLog?.path===join(root,"logs/backend.log")&&Number.isSafeInteger(value.backendLog?.dev)&&Number.isSafeInteger(value.backendLog?.ino);if(!baseValid||(value.stage==="READY"?!readyLog:value.backendLog!==null))throw new Error("manual ownership identity mismatch");return value;}
 async function run(executable,argv,options={}){return await exec(executable,argv,{...options,maxBuffer:2*1024*1024,encoding:"utf8"});}
-function descriptor(id,source){return{workspace:{schema_version:4,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
+function descriptor(id,source){return{workspace:{schema_version:3,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},semantic_index:{vector_store:{engine:"qdrant",collection:id,dimensions:1024,distance:"cosine"},embedding:{provider:"ollama_internal",model:"qwen3-embedding:0.6b",dimensions:1024}},llm_policy:{allowed:["zai/glm-5.2"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
 function descriptors(){return[descriptor("p1-filesystem",{type:"filesystem",uri:"workspace-content/p1-filesystem/evidence",patterns:["**/*.md"],max_bytes:10485760}),descriptor("p1-http",{type:"http",uris:["https://evidence.example.test/guide.md"],authentication:"signed_urls_file",connect_timeout_ms:1250,read_timeout_ms:30001,max_bytes:12345,max_redirects:2,allow_private_hosts:false,max_cache_bytes:67890}),descriptor("p1-s3",{type:"s3",uri:"s3://p1-evidence/published/",endpoint_url:"https://s3.example.test/",region:"eu-west-1",credentials:"static_files",trusted_endpoint:true,allow_private_endpoint:false,allow_insecure_endpoint:false,max_bytes:12345,max_objects:33,max_pages:4,page_size:5})];}
 function quote(value){return `'${String(value).replaceAll("'",`'"'"'`)}'`;}
 async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof","python3"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}}

backend/scripts/p1-manual-acceptance.test.mjs

@@ -403,7 +403,7 @@ test("generated render command validates saved responses and owned snapshot befo
 });
 
 const renderSnapshotYaml=`workspace:
-  schema_version: 4
+  schema_version: 3
   id: p1-filesystem
   name: P1 filesystem
   language: en
@@ -412,6 +412,11 @@ dwh:
   database: postgres
   schema: public
   supported_transports: [postgres_direct]
+semantic_index:
+  vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
+  embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
+llm_policy:
+  allowed: [zai/glm-5.2]
 evidence:
   source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
   policy: {max_chunk_chars: 4000, retain_published_generations: 3}

backend/scripts/p1-render-snapshot.test.mjs

@@ -16,7 +16,7 @@ async function fixture() {
  await writeFile(join(root,"installation/base.yaml"),"{}\n");
  const secret=join(root,"fixture-secrets/dwh-password"); await writeFile(secret,"not-inspected",{mode:0o600});
  await writeFile(snapshot,`workspace:
-  schema_version: 4
+  schema_version: 3
   id: p1-filesystem
   name: P1 filesystem
   language: en
@@ -25,6 +25,11 @@ dwh:
   database: postgres
   schema: public
   supported_transports: [postgres_direct]
+semantic_index:
+  vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
+  embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
+llm_policy:
+  allowed: [zai/glm-5.2]
 evidence:
   source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
   policy: {max_chunk_chars: 4000, retain_published_generations: 3}
@@ -56,7 +61,7 @@ test("renderer refuses snapshot manifest head, digest, and expected-digest tampe
 
 test("renderer refuses a missing or malformed snapshot manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/nomanifest.yaml"); await rm(f.manifestPath); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*(missing|unbounded|unsafe)/); await writeFile(f.manifestPath,"{not json"); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*malformed/); await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); });
 
-test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n  schema_version: 4\n  id: p1-filesystem\n  name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); });
+test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n  schema_version: 3\n  id: p1-filesystem\n  name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); });
 
 test("renderer anchors publication when rendered parent is concurrently swapped", async()=>{
  const f=await fixture(),output=join(f.root,"rendered/raced.yaml"),moved=join(f.root,"rendered-moved"),outside=join(f.repo,"outside-rendered"); await mkdir(outside);

backend/scripts/p11-acceptance.mjs

@@ -328,8 +328,13 @@ async function tht(ctx, argv, options = {}) {
 function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
 function baseWorkspace(id, evidenceSource) {
   return {
-    workspace: { schema_version: 4, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
   };
 }

backend/scripts/p11-manual-acceptance.mjs

@@ -81,8 +81,13 @@ async function git(executable, argv, options = {}) {
 function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
 function baseWorkspace(id, evidenceSource) {
   return {
-    workspace: { schema_version: 4, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
   };
 }

backend/scripts/p2-acceptance.mjs

@@ -375,11 +375,16 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p2p6-acceptance.mjs

@@ -376,11 +376,16 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p3-acceptance.mjs

@@ -379,11 +379,16 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p5-acceptance.mjs

@@ -374,11 +374,16 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/p6-acceptance.mjs

@@ -374,11 +374,16 @@ function installationProjectName(installationPath) {
 
 function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) {
   return {
-    workspace: { schema_version: 4, id, name: `P2 ${id}`, language: "en" },
+    workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" },
     dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] },
+    semantic_index: {
+      vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
+      embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+    },
     diagnostics: {
       dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } },
     },
+    llm_policy: { allowed: ["zai/glm-5.2"] },
     ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}),
   };
 }

backend/scripts/verify-workspace-descriptor-files.mjs

@@ -22,7 +22,6 @@ const reviewedExpandableBlocks = new Map([
     { sha256: "37f18ce7ce93cb8b84f3b3708462cc16d50fdc7bab22836c382dbacf8382f05f", rationale: "Generates the reviewed synthetic tht installer artifact." },
   ]],
   ["scripts/test-server-pi-state-topology.sh", [
-    { sha256: "435c769b8cbd7b834f56fdabddb86ba04fb404dd0d8a6b7c21719a8b0f7cf011", rationale: "Generates the reviewed model-catalog projection override for the isolated server topology test." },
     { sha256: "6ae9567db53d6cd45a2c19c98acaf45f382450b157ea7d6f6d35125f68c50947", rationale: "Generates the isolated server topology test environment, including its installation descriptor and authentication configuration root." },
   ]],
   ["scripts/test-vector-backup-restore-safety.sh", [
@@ -37,10 +36,11 @@ const reviewedExpandableBlocks = new Map([
     { sha256: "b903e5dae953ae1372f1a5276f12a92ed3dd632b897f3afe5e00c646d90a1b42", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
   ]],
   ["scripts/unified-deployment-smoke.sh", [
-    { sha256: "b6c0826151b2c8b955399d1abf5b691cc8fe6b6454b17da000dde7ba3bc55d2d", rationale: "Generates the reviewed local Task 13 Compose override with normalized catalog mounts." },
-    { sha256: "24f69d12b8554aa2bebba455be99fde3e60743eef5a40fa2ef5b29397a477c03", rationale: "Generates the reviewed local Task 13 installation descriptor with its model catalog." },
+    { sha256: "1d60bf140165a8fabfa0c3729e776136904717e67becf3e0ab68c70d8e37847e", rationale: "Generates reviewed Task 13 runtime configuration." },
+    { sha256: "36d3d8a2362dbdc4fad90948d6c227586d749f56b9a4bc5b6b5a91bcbec6407b", rationale: "Generates the reviewed local Task 13 Compose override." },
+    { sha256: "c556f7d910d0788e219b042957e6b307cb9925b43920c680535d0d3a6dcbdb25", rationale: "Generates the reviewed local Task 13 installation descriptor." },
     { sha256: "526006fa6d48a8080b3834723630c64de5005a67243e944ebf1da15212b4d654", rationale: "Generates the reviewed server Task 13 Compose override." },
-    { sha256: "406ccead1967f642225c946fc4a23fe5b019c9764cc5153e1125876ade16ec90", rationale: "Generates the reviewed projected-auth server Task 13 installation descriptor with its model catalog." },
+    { sha256: "c57ae2205c21ead0c2015a353aaabb948fa4ddd9b78a2cdcdb71f48cf2db742d", rationale: "Generates the reviewed projected-auth server Task 13 installation descriptor." },
   ]],
   ["scripts/vector-backup.sh", [
     { sha256: "571899db49dfdcec8107fbe1e0a86a61e7581979d3c4c248c20546843e275bcf", rationale: "Generates the reviewed backup manifest inside the helper command." },
@@ -103,7 +103,6 @@ function isPolicyImplementationException(label, category) {
   ]);
   if (implementations.has(label)) return true;
   if (category === "migration-marker" && new Set([
-    "backend/src/workspaces/schema.ts",
     "scripts/workspace_descriptor_doc_contract.py",
     "scripts/test_workspace_descriptor_doc_contract.py",
     "backend/scripts/clean-dist.test.mjs",
@@ -174,7 +173,7 @@ function validateWorkspaceSource(source, label, { requireWorkspace, expandable =
   try {
     parseWorkspaceYaml(source);
   } catch (error) {
-    throw new Error(`${label}: workspace descriptor is not valid schema v4: ${error instanceof Error ? error.message : String(error)}`);
+    throw new Error(`${label}: workspace descriptor is not valid schema v3: ${error instanceof Error ? error.message : String(error)}`);
   }
   return true;
 }

backend/scripts/verify-workspace-descriptor-files.test.mjs

@@ -33,20 +33,20 @@ function bashN(root, path) {
 function replaceWorkspaceKeys(source, workspaceKey, schemaLine) {
   return source
     .replace(/^workspace:$/m, workspaceKey)
-    .replace(/^  schema_version: 4$/m, schemaLine);
+    .replace(/^  schema_version: 3$/m, schemaLine);
 }
 
-test("production parser accepts semantic v4 with quoted Unicode/tagged keys and spacing", async (t) => {
+test("production parser accepts semantic v3 with quoted Unicode/tagged keys and spacing", async (t) => {
   const root = await fixture(t);
   const unicode = replaceWorkspaceKeys(
     canonicalDescriptor,
     '"\\u0077orkspace" :',
-    '  "\\u0073chema_version" : 4',
+    '  "\\u0073chema_version" : 3',
   );
   const tagged = replaceWorkspaceKeys(
     canonicalDescriptor,
     "!!str workspace :",
-    "  !!str schema_version : 4",
+    "  !!str schema_version : 3",
   );
   await put(root, "deploy/workspaces/unicode.yaml", unicode);
   await put(root, "deploy/workspaces/tagged.yaml", tagged);
@@ -59,15 +59,14 @@ test("production parser accepts semantic v4 with quoted Unicode/tagged keys and
   });
 });
 
-test("production parser rejects fancy keys with every non-v4 or ambiguous value", async (t) => {
+test("production parser rejects fancy keys with every non-v3 or ambiguous value", async (t) => {
   const invalid = [
     ["unicode-v2", '"\\u0077orkspace" :', '  "\\u0073chema_version" : 2'],
-    ["unicode-v3", '"\\u0077orkspace" :', '  "\\u0073chema_version" : 3'],
-    ["tagged-leading-zero", "!!str workspace :", "  !!str schema_version : 03"],
-    ["hexadecimal", "workspace :", "  schema_version : 0x3"],
-    ["multiline", "workspace :", "  schema_version : >\n    4"],
-    ["duplicate", "workspace :", "  schema_version : 4\n  schema_version: 4"],
-    ["inline", "workspace: { schema_version: 4 }", "  schema_version: 4"],
+    ["tagged-leading-zero", "!!str workspace :", "  !!str schema_version : 02"],
+    ["hexadecimal", "workspace :", "  schema_version : 0x2"],
+    ["multiline", "workspace :", "  schema_version : >\n    3"],
+    ["duplicate", "workspace :", "  schema_version : 3\n  schema_version: 3"],
+    ["inline", "workspace: { schema_version: 3 }", "  schema_version: 3"],
   ];
   for (const [name, workspaceKey, schemaLine] of invalid) {
     await t.test(name, async () => {
@@ -121,7 +120,7 @@ test("PowerShell embedded workspace mappings are rejected while bundle-only stri
   const root = await fixture(t);
   const source = [
     "$workspace = @'",
-    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 0x2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 0x2").trimEnd(),
     "'@",
     '$bundle = @"',
     "bundle:",
@@ -138,7 +137,7 @@ test("PowerShell embedded workspace mappings are rejected while bundle-only stri
 
 test("workspace descriptor family entries require a top-level workspace", async (t) => {
   const root = await fixture(t);
-  await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n  schema_version: 4\n");
+  await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n  schema_version: 3\n");
   await assert.rejects(
     verifyEntries({
       root,
@@ -180,7 +179,7 @@ test("script scalar workspace remains a bundle even with descriptor-like sibling
 test("standalone descriptor files require workspace to be a mapping", async (t) => {
   const root = await fixture(t);
   const path = "scripts/fixtures/workspace-registry-scalar.yaml";
-  await put(root, path, "workspace: analytics\nschema_version: 4\n");
+  await put(root, path, "workspace: analytics\nschema_version: 3\n");
   await assert.rejects(
     verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }),
     /workspace.*mapping/i,
@@ -194,11 +193,11 @@ test("Bash extractor supports hyphen, digit, escaped delimiters, and tab strippi
       name: "hyphen-v2",
       opener: "cat <<'WORKSPACE-YAML'",
       delimiter: "WORKSPACE-YAML",
-      descriptor: canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2"),
+      descriptor: canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2"),
       rejected: true,
     },
     {
-      name: "digit-v4",
+      name: "digit-v3",
       opener: "cat <<2YAML",
       delimiter: "2YAML",
       descriptor: canonicalDescriptor,
@@ -208,11 +207,11 @@ test("Bash extractor supports hyphen, digit, escaped delimiters, and tab strippi
       name: "escaped-v2",
       opener: "cat <<WORKSPACE\\-YAML",
       delimiter: "WORKSPACE-YAML",
-      descriptor: canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2"),
+      descriptor: canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2"),
       rejected: true,
     },
     {
-      name: "tab-strip-v4",
+      name: "tab-strip-v3",
       opener: "cat <<-'TAB-YAML'",
       delimiter: "\tTAB-YAML",
       descriptor: canonicalDescriptor.split("\n").map((line) => `\t${line}`).join("\n"),
@@ -273,7 +272,7 @@ test("non-stripping heredoc close requires an exact physical delimiter line", as
     "#!/usr/bin/env bash",
     "cat <<'---'",
     "---   ",
-    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
     "---",
     "",
   ].join("\n");
@@ -314,7 +313,7 @@ test("double-quoted non-special backslash is preserved in the delimiter", async
     "#!/usr/bin/env bash",
     'cat <<"\\---"',
     "---",
-    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
     "\\---",
     "",
   ].join("\n");
@@ -356,7 +355,7 @@ test("split heredoc operator continuation cannot bypass v2 validation", async (t
     "#!/usr/bin/env bash",
     "cat <\\",
     "<'YAML'",
-    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
     "YAML",
     "",
   ].join("\n");
@@ -425,7 +424,7 @@ test("PowerShell comment backslash cannot hide a following v2 here-string", asyn
   const source = [
     "# harmless PowerShell comment \\",
     "$workspace = @'",
-    canonicalDescriptor.replace("  schema_version: 4", "  schema_version: 2").trimEnd(),
+    canonicalDescriptor.replace("  schema_version: 3", "  schema_version: 2").trimEnd(),
     "'@",
     "",
   ].join("\n");
@@ -436,7 +435,7 @@ test("PowerShell comment backslash cannot hide a following v2 here-string", asyn
   );
 });
 
-test("PowerShell dialect accepts normal v4 and non-workspace bundle here-strings", async (t) => {
+test("PowerShell dialect accepts normal v3 and non-workspace bundle here-strings", async (t) => {
   const root = await fixture(t);
   const path = "scripts/powershell-valid-smoke.ps1";
   const source = [
@@ -495,10 +494,10 @@ test("PowerShell cast and concatenation openers cannot hide embedded descriptors
 test("expandable YAML interpolation that can hide a workspace descriptor fails closed", async (t) => {
   const root = await fixture(t);
   const cases = [
-    ["braced-key", "${key}:\n  schema_version: 4"],
-    ["plain-key", "$key:\n  schema_version: 4"],
-    ["quoted-key", '"$key" :\n  schema_version: 4'],
-    ["subexpression-key", "$($key):\n  schema_version: 4"],
+    ["braced-key", "${key}:\n  schema_version: 3"],
+    ["plain-key", "$key:\n  schema_version: 3"],
+    ["quoted-key", '"$key" :\n  schema_version: 3'],
+    ["subexpression-key", "$($key):\n  schema_version: 3"],
     ["version", "workspace:\n  schema_version: $version"],
   ];
   for (const [name, body] of cases) {
@@ -565,7 +564,7 @@ test("unmarked expandable Bash YAML cannot generate descriptor keys or values at
       "key=workspace",
       "cat <<YAML",
       generatedKey,
-      "  schema_version: 4",
+      "  schema_version: 3",
       "YAML",
       "",
     ].join("\n");
@@ -601,14 +600,14 @@ test("an in-band marker cannot authorize expandable content", async (t) => {
   for (const [path, source] of [
     ["scripts/fake-marker.sh", [
       "#!/usr/bin/env bash",
-      "# schema-v4-only: expandable-nonworkspace",
+      "# schema-v3-only: expandable-nonworkspace",
       "cat <<YAML",
       "${DESCRIPTOR}",
       "YAML",
       "",
     ].join("\n")],
     ["scripts/fake-marker.ps1", [
-      "# schema-v4-only: expandable-nonworkspace",
+      "# schema-v3-only: expandable-nonworkspace",
       '$yaml = @"',
       "$descriptor",
       '"@',

backend/src/app.ts

@@ -3,7 +3,6 @@ import cors from "@fastify/cors";
 import cookie from "@fastify/cookie";
 import rateLimit from "@fastify/rate-limit";
 import { dirname, isAbsolute, join } from "node:path";
-import { fileURLToPath } from "node:url";
 import { tmpdir } from "node:os";
 import type { AppConfig } from "./config.js";
 import { ThtRunner } from "./tht/tht-runner.js";
@@ -23,8 +22,7 @@ import { isUsableAuthenticationSecret } from "./auth/secret-policy.js";
 import { secretValue } from "./config/secret-bundle.js";
 import { sessionRoutes } from "./routes/sessions.js";
 import { sqlRoutes } from "./routes/sql.js";
-import { metaRoutes } from "./routes/meta.js";
-import type { ListModelsFn } from "./pi/list-models.js";
+import { metaRoutes, type ListModelsFn } from "./routes/meta.js";
 import { settingsRoutes, effectiveSettings } from "./routes/settings.js";
 import { createPiModelLister } from "./pi/list-models.js";
 import { createPiManagement, type PiManagementService } from "./pi/management.js";
@@ -33,11 +31,7 @@ import { ReadinessManager } from "./runtime/readiness-manager.js";
 import { MaintenanceBarrier } from "./runtime/maintenance-gate.js";
 import { WorkspaceRegistry } from "./workspaces/registry.js";
 import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js";
-import {
-  workspaceRoutes,
-  type WorkspaceDatabaseTester,
-  type WorkspaceDiagnoser,
-} from "./routes/workspaces.js";
+import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js";
 import { piManagementRoutes } from "./routes/pi-management.js";
 import { supportsSessionRuntime } from "./workspaces/bindings.js";
 import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js";
@@ -62,11 +56,8 @@ import { metadataGenerationModelRoutes } from "./routes/metadata-generation-mode
 import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-description-consolidation.js";
 import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js";
 import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js";
-import { SensitivityAnalysisService } from "./catalog/sensitivity-analysis-service.js";
-import { SensitivityAnalysisRunner } from "./catalog/sensitivity-analysis-runner.js";
-import { SensitivityClassifier, type LocalNerDetector, type SensitivityValueSource } from "./catalog/sensitivity-classifier.js";
-import { ConcreteSensitivityValueSource } from "./catalog/sensitivity-value-source.js";
-import { PythonLocalNerDetector } from "./catalog/local-ner-detector.js";
+import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js";
+import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js";
 import {
   ConcreteDescriptionSourceSampler,
   type DescriptionSourceSampler,
@@ -75,11 +66,6 @@ import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description
 import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js";
 import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
 import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
-import { loadRuntimeModelCatalog, type RuntimeModelCatalog } from "./models/runtime-model-catalog.js";
-import { createProductionWorkspacePreprocessingService } from "./workspace-maintenance.js";
-import type { WorkspacePreprocessingService } from "./workspaces/preprocessing-service.js";
-import { PreprocessingStateStore } from "./workspaces/preprocessing-state.js";
-import { workspacePreprocessingRoutes } from "./routes/workspace-preprocessing.js";
 
 export interface BuildAppDeps {
   thtRunner?: ThtRunner;
@@ -91,9 +77,7 @@ export interface BuildAppDeps {
   hub?: SseHub;
   workspaceRegistry?: WorkspaceRegistry;
   workspaceDiagnoser?: WorkspaceDiagnoser;
-  workspaceDatabaseTester?: WorkspaceDatabaseTester;
   workspaceSecretStore?: WorkspaceSecretStore;
-  workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear">;
   catalogRepository?: CatalogRepository;
   catalogService?: CatalogService;
   catalogPostgresAccess?: CatalogPostgresAccess;
@@ -104,11 +88,8 @@ export interface BuildAppDeps {
   catalogSyncWorker?: CatalogSyncWorker;
   catalogOperationCoordinator?: CatalogOperationCoordinator;
   metadataGenerationModels?: MetadataGenerationModels;
-  runtimeModelCatalog?: RuntimeModelCatalog;
   modelCompleter?: ModelCompleter;
   descriptionSourceSampler?: DescriptionSourceSampler;
-  sensitivityValueSource?: SensitivityValueSource;
-  localNerDetector?: LocalNerDetector;
   workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
   maintenanceBarrier?: MaintenanceBarrier;
   piManagement?: PiManagementService;
@@ -161,8 +142,6 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   app.register(cookie);
   app.register(rateLimit, { global: false });
 
-  const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
-  const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
   const tht = deps?.thtRunner ?? new ThtRunner({
     thtBin: config.thtBin,
     harnessDir: config.harnessDir,
@@ -173,32 +152,20 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     secretsFile: config.secretsFile,
     secretFiles: config.secretFiles,
     workspaceSecretStore,
-    catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
     semanticRuntime: {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
-      internalEmbeddingId: config.internalEmbeddingId,
       internalEmbeddingModel: config.internalEmbeddingModel,
       internalEmbeddingDimensions: config.internalEmbeddingDimensions,
     },
   });
-  const workspacePreprocessingService = deps?.workspacePreprocessingService
-    ?? createProductionWorkspacePreprocessingService({
-      config,
-      catalogRepository,
-      registry: workspaceRegistry,
-      workspaceSecretStore,
-      runner: tht as ThtRunner,
-    });
+  const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
   const hub = deps?.hub ?? new SseHub();
+  const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
+  const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
   const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
-  const runtimeModelCatalog = deps?.runtimeModelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
-  const mgr = deps?.mgr ?? new PiProcessManager(config, {
-    ...(deps?.spawnFn ? { spawnFn: deps.spawnFn } : {}),
-    modelCatalog: runtimeModelCatalog,
-  });
   const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
-    catalogFile: config.modelCatalogFile,
+    installationFile: config.installationConfigFile,
     secretsFile: config.secretsFile,
   });
   const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
@@ -219,24 +186,12 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     catalogOperationCoordinator,
     descriptionSourceSampler,
   );
-  const sensitivityValueSource = deps?.sensitivityValueSource
-    ?? new ConcreteSensitivityValueSource(catalogPostgresAccess, workspaceSecretStore);
-  const configuredNerWorker = config.sensitivityNer?.workerScript
-    ?? fileURLToPath(new URL("../python/sensitivity_ner_worker.py", import.meta.url));
-  const localNerDetector = deps?.localNerDetector ?? (config.sensitivityNer
-    ? new PythonLocalNerDetector({
-      pythonExecutable: config.sensitivityNer.pythonExecutable,
-      workerScript: configuredNerWorker,
-      modelPath: config.sensitivityNer.modelPath,
-      cwd: dirname(configuredNerWorker),
-      threads: config.sensitivityNer.threads,
-    })
-    : undefined);
-  const sensitiveDataSuggester = new SensitivityAnalysisService(
+  const sensitiveDataSuggester = new SensitiveDataSuggester(
     catalogRepository,
-    new SensitivityClassifier(sensitivityValueSource, localNerDetector),
+    metadataGenerationModels,
+    modelCompleter,
   );
-  const sensitivityAnalysisRunner = new SensitivityAnalysisRunner(
+  const sensitiveDataSuggestionRunner = new SensitiveDataSuggestionRunner(
     catalogRepository,
     sensitiveDataSuggester,
   );
@@ -249,13 +204,17 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     catalogPostgresAccess,
     catalogOperationCoordinator,
   );
-  const workspaceDatabaseTester = deps?.workspaceDatabaseTester ?? (async (workspaceId: string) => {
-    const database = await catalogRepository.getByWorkspace(workspaceId);
-    return database ? catalogService.test(database) : undefined;
-  });
   const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
   const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
     ?? new CatalogLogicalRelationshipService(catalogRepository);
+  const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider
+    ?? (config.catalogDatabase === undefined
+      ? undefined
+      : new EffectiveRelationshipSnapshotProvider(
+        catalogRepository,
+        catalogLogicalRelationshipService,
+        catalogOperationCoordinator,
+      ));
   const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
     catalogPostgresAccess,
     workspaceSecretStore,
@@ -268,25 +227,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   );
   app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
   app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
-  app.addHook("onReady", async () => { await sensitivityAnalysisRunner.initialize(); });
-  if (localNerDetector?.warmup) {
-    app.addHook("onReady", async () => {
-      void localNerDetector.warmup?.().catch(() => undefined);
-    });
-  }
+  app.addHook("onReady", async () => { await sensitiveDataSuggestionRunner.initialize(); });
   if (!deps?.catalogRepository && catalogRepository.close) {
     app.addHook("onClose", async () => { await catalogRepository.close?.(); });
   }
   app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
   app.addHook("onClose", async () => { await descriptionGenerationWorker.stop(); });
-  if (localNerDetector?.close) {
-    app.addHook("onClose", async () => { await localNerDetector.close?.(); });
-  }
   const workspaceDiagnoser = deps?.workspaceDiagnoser
     ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
-      internalEmbeddingId: config.internalEmbeddingId,
       internalEmbeddingModel: config.internalEmbeddingModel,
       internalEmbeddingDimensions: config.internalEmbeddingDimensions,
     });
@@ -309,7 +259,6 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   );
 
   const listModels = deps?.listModels ?? createPiModelLister(config, {
-    modelCatalog: runtimeModelCatalog,
     warn: (detail) => app.log.warn(
       { component: "pi-model-list", detail },
       "Pi enabled-model configuration warning",
@@ -322,7 +271,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   const getSettings = async (principal: PrincipalContext): Promise<Settings> => {
     if (deps?.getSettings) return await deps.getSettings(principal);
     const stored = loadSettings(config);
-    const effective = effectiveSettings(config, stored, runtimeModelCatalog);
+    const effective = effectiveSettings(config, stored);
     // In the registry system the legacy `harness/workspaces/*.yaml` default is obsolete: when no
     // installation workspace is pinned, default to the first active registry workspace.
     if (!stored.workspace) {
@@ -335,9 +284,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     }
     return effective;
   };
-  const piManagement = deps?.piManagement ?? createPiManagement(config, {
-    modelCatalog: runtimeModelCatalog,
-  });
+  const piManagement = deps?.piManagement ?? createPiManagement(config, { listModels });
 
   const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile);
   const localRegistryResolver = deps?.localUserRegistry === undefined
@@ -461,9 +408,8 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     dwhPrecheck: config.dwhPrecheck,
     legacyWorkspaceMode: config.legacyWorkspaceMode,
     workspaceRuntimeSupport,
-    modelCatalog: runtimeModelCatalog,
     maintenanceBarrier,
-    catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
+    effectiveRelationships,
   });
   app.post("/internal/maintenance/activate", async (req, reply) => {
     try {
@@ -493,24 +439,13 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     return maintenanceBarrier.status();
   });
   sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
-  metaRoutes(app, { harnessDir: config.harnessDir, modelCatalog: runtimeModelCatalog });
+  metaRoutes(app, { harnessDir: config.harnessDir, listModels });
   workspaceRoutes(app, {
     registry: workspaceRegistry,
     config: config.workspaceRegistry,
     diagnose: workspaceDiagnoser,
     authDiagnoser,
     secretStore: workspaceSecretStore,
-    testDatabaseConnection: workspaceDatabaseTester,
-  });
-  workspacePreprocessingRoutes(app, {
-    repository: catalogRepository,
-    registry: workspaceRegistry,
-    service: workspacePreprocessingService,
-    inputFingerprint: tht as ThtRunner,
-    readLatestJob: (workspaceId) => new PreprocessingStateStore({
-      dataRoot: config.dataRoot ?? "/data",
-      workspaceId,
-    }).readLatestJob(),
   });
   catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
   catalogTableRoutes(app, {
@@ -535,9 +470,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   catalogDescriptionGenerationRoutes(app, {
     repository: catalogRepository,
     worker: descriptionGenerationWorker,
-    sensitivityAnalysisRunner,
+    sensitiveDataSuggestionRunner,
   });
-  settingsRoutes(app, { cfg: config, getSettings });
+  settingsRoutes(app, { cfg: config, listModels, getSettings });
   piManagementRoutes(app, { service: piManagement });
 
   return app;

backend/src/catalog/description-generation-worker.ts

@@ -59,16 +59,10 @@ const completionResponseSchema = z.object({
 class InvalidModelJsonError extends Error {}
 class InvalidModelSchemaError extends Error {}
 class MissingModelTargetsError extends Error {}
-
-interface DescriptionGenerationFailureTarget {
-  id: string;
-  reference: string;
-}
-
 class DescriptionGenerationBatchError extends Error {
   constructor(
     readonly failure: unknown,
-    readonly failedTargets: readonly DescriptionGenerationFailureTarget[],
+    readonly failedTargets: readonly { id: string; label: "Catalog Column" | "Catalog Table" }[],
   ) {
     super("description generation batch failed");
   }
@@ -144,7 +138,7 @@ type ParsedOutcome = z.infer<typeof outcomeSchema>;
 
 interface DescriptionGenerationPlan {
   columnTargets: SelectedColumnTarget[];
-  tableTargets: Array<{ id: string; name: string }>;
+  tableIds: string[];
 }
 
 interface DescriptionGenerationCounters {
@@ -170,17 +164,10 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
   };
 }
 
-function targetReference(target: SelectedTarget): string {
+function failureTarget(target: SelectedTarget) {
   return target.kind === "column"
-    ? `Column ${JSON.stringify(`${target.table.name}.${target.column.name}`)}`
-    : `Table ${JSON.stringify(target.table.name)}`;
-}
-
-function failureTarget(target: SelectedTarget): DescriptionGenerationFailureTarget {
-  return {
-    id: target.kind === "column" ? target.column.id : target.table.id,
-    reference: targetReference(target),
-  };
+    ? { id: target.column.id, label: "Catalog Column" as const }
+    : { id: target.table.id, label: "Catalog Table" as const };
 }
 
 const NON_GENERATABLE_DESCRIPTION: Record<DescriptionGenerationRun["language"], string> = {
@@ -694,7 +681,7 @@ function safeFailure(error: unknown): string {
 function batchFailureEvent(error: DescriptionGenerationBatchError): string {
   const summary = safeFailure(error);
   return error.failedTargets.length > 0
-    ? `${summary} Affected target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.reference).join(", ")}.`
+    ? `${summary} Affected ${error.failedTargets[0]!.label} target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.id).join(", ")}.`
     : summary;
 }
 
@@ -807,7 +794,7 @@ export class DescriptionGenerationWorker {
           scope === "selected_columns" ? "column" : "table",
         );
       }
-      const total = plan.columnTargets.length + plan.tableTargets.length;
+      const total = plan.columnTargets.length + plan.tableIds.length;
       if (total === 0 && (scope === "all" || scope === "missing")) {
         throw new DescriptionGenerationNoEligibleTargetsError(scope);
       }
@@ -947,18 +934,12 @@ export class DescriptionGenerationWorker {
     };
     await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
     throwIfCancelled(signal);
-    if (plan.tableTargets.length > 0) {
-      const tableTargets = await this.resolveTableTargets(
-        run.databaseId,
-        plan.tableTargets.map((target) => target.id),
-      );
+    if (plan.tableIds.length > 0) {
+      const tableTargets = await this.resolveTableTargets(run.databaseId, plan.tableIds);
       if (!tableTargets) {
         throw new DescriptionGenerationBatchError(
           new Error("selected tables changed during generation"),
-          plan.tableTargets.map((target) => ({
-            id: target.id,
-            reference: `Table ${JSON.stringify(target.name)}`,
-          })),
+          plan.tableIds.map((id) => ({ id, label: "Catalog Table" })),
         );
       }
       await this.processTargets(run, database, tableTargets, model, counters, signal);
@@ -1112,8 +1093,8 @@ export class DescriptionGenerationWorker {
           run.id,
           "info",
           outcome.outcome === "generated"
-            ? `Generated description for ${targetReference(target)}.`
-            : `Stored non-generatable result for ${targetReference(target)}.`,
+            ? `Generated description for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`
+            : `Stored non-generatable result for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`,
         );
       }
     }
@@ -1207,22 +1188,16 @@ export class DescriptionGenerationWorker {
       }
       return {
         columnTargets,
-        tableTargets: tables
+        tableIds: tables
           .filter((table) => scope === "all" || !table.generatedDescription?.trim())
-          .map((table) => ({ id: table.id, name: table.name })),
+          .map((table) => table.id),
       };
     }
     if (scope === "selected_tables") {
       const tableById = new Map(tables.map((table) => [table.id, table]));
       const selected = targetIds.map((tableId) => tableById.get(tableId));
       if (selected.some((table) => table === undefined)) return undefined;
-      return {
-        columnTargets: [],
-        tableTargets: (selected as CatalogTable[]).map((table) => ({
-          id: table.id,
-          name: table.name,
-        })),
-      };
+      return { columnTargets: [], tableIds: [...targetIds] };
     }
 
     const byId = new Map<string, SelectedColumnTarget>();
@@ -1234,7 +1209,7 @@ export class DescriptionGenerationWorker {
     const targets = targetIds.map((columnId) => byId.get(columnId));
     return targets.some((target) => target === undefined)
       ? undefined
-      : { columnTargets: targets as SelectedColumnTarget[], tableTargets: [] };
+      : { columnTargets: targets as SelectedColumnTarget[], tableIds: [] };
   }
 
   private async resolveTableTargets(

backend/src/catalog/local-ner-detector.ts

@@ -1,254 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
-import { tmpdir } from "node:os";
-import { z } from "zod";
-import type {
-  LocalNerCandidate,
-  LocalNerDetector,
-  LocalNerEvidence,
-} from "./sensitivity-classifier.js";
-
-const MAX_LINE_BYTES = 64 * 1024;
-const candidateSchema = z.object({
-  columnId: z.uuid(),
-  text: z.string().min(1).max(500),
-}).strict();
-const workerMessageSchema = z.union([
-  z.object({ ready: z.literal(true) }).strict(),
-  z.object({
-    id: z.uuid(),
-    ok: z.literal(true),
-    evidence: z.array(z.object({
-      columnId: z.uuid(),
-      label: z.string().min(1).max(80),
-      confidence: z.number().min(0).max(1),
-    }).strict()).max(1_000),
-  }).strict(),
-  z.object({ id: z.uuid(), ok: z.literal(false), error: z.string().min(1).max(80) }).strict(),
-]);
-
-export class LocalNerUnavailableError extends Error {
-  constructor() {
-    super("local NER is unavailable");
-    this.name = "LocalNerUnavailableError";
-  }
-}
-
-interface PendingRequest {
-  resolve: (value: readonly LocalNerEvidence[]) => void;
-  reject: (error: Error) => void;
-  timer: ReturnType<typeof setTimeout>;
-  signal: AbortSignal;
-  cancel: () => void;
-}
-
-/** Persistent JSONL adapter for the optional, CPU-only Python NER worker. */
-export class PythonLocalNerDetector implements LocalNerDetector {
-  private child?: ChildProcessWithoutNullStreams;
-  private ready?: Promise<void>;
-  private readyResolve?: () => void;
-  private readyReject?: (error: Error) => void;
-  private workerReady = false;
-  private stdout = "";
-  private readonly pending = new Map<string, PendingRequest>();
-
-  constructor(private readonly options: {
-    pythonExecutable: string;
-    workerScript: string;
-    modelPath: string;
-    cwd: string;
-    threads?: number;
-    startupTimeoutMs?: number;
-  }) {}
-
-  async warmup(): Promise<void> {
-    await this.ensureStarted();
-  }
-
-  isReady(): boolean {
-    return this.workerReady
-      && this.child !== undefined
-      && this.child.exitCode === null
-      && this.child.signalCode === null;
-  }
-
-  async detect(
-    candidates: readonly LocalNerCandidate[],
-    signal: AbortSignal,
-    deadline: number,
-  ): Promise<readonly LocalNerEvidence[]> {
-    const parsed = z.array(candidateSchema).min(1).max(128).parse(candidates);
-    if (signal.aborted || deadline <= Date.now()) throw new LocalNerUnavailableError();
-    await this.ensureStartedWithin(signal, deadline);
-    if (!this.child || this.child.exitCode !== null || this.child.signalCode !== null) {
-      throw new LocalNerUnavailableError();
-    }
-    const id = randomUUID();
-    return await new Promise<readonly LocalNerEvidence[]>((resolve, reject) => {
-      const fail = () => {
-        this.finishPending(id);
-        reject(new LocalNerUnavailableError());
-        this.stopWorker();
-      };
-      const timer = setTimeout(fail, Math.max(1, Math.floor(deadline - Date.now())));
-      const cancel = fail;
-      const pending: PendingRequest = { resolve, reject, timer, signal, cancel };
-      this.pending.set(id, pending);
-      signal.addEventListener("abort", cancel, { once: true });
-      this.child!.stdin.write(`${JSON.stringify({ id, candidates: parsed })}\n`, (error) => {
-        if (error) fail();
-      });
-    });
-  }
-
-  async close(): Promise<void> {
-    const child = this.child;
-    if (!child || child.exitCode !== null || child.signalCode !== null) return;
-    await new Promise<void>((resolve) => {
-      child.once("close", () => resolve());
-      child.kill("SIGTERM");
-      setTimeout(() => {
-        if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
-      }, 250).unref();
-    });
-  }
-
-  private async ensureStarted(): Promise<void> {
-    if (this.ready) return await this.ready;
-    this.ready = new Promise<void>((resolve, reject) => {
-      this.readyResolve = resolve;
-      this.readyReject = reject;
-    });
-    const threads = String(this.options.threads ?? 2);
-    const inheritedRuntimeEnvironment = Object.fromEntries([
-      "PATH", "SystemRoot", "WINDIR", "PATHEXT", "TMPDIR", "TEMP", "TMP", "LANG", "LC_ALL",
-    ].flatMap((name) => process.env[name] === undefined ? [] : [[name, process.env[name]!]]));
-    const child = spawn(this.options.pythonExecutable, [
-      "-I",
-      "-B",
-      this.options.workerScript,
-      "--model",
-      this.options.modelPath,
-      "--threads",
-      threads,
-    ], {
-      cwd: this.options.cwd,
-      stdio: ["pipe", "pipe", "pipe"],
-      env: {
-        ...inheritedRuntimeEnvironment,
-        HOME: process.env.HOME ?? tmpdir(),
-        CUDA_VISIBLE_DEVICES: "",
-        HIP_VISIBLE_DEVICES: "",
-        HF_HUB_OFFLINE: "1",
-        HF_HUB_DISABLE_TELEMETRY: "1",
-        TRANSFORMERS_OFFLINE: "1",
-        TOKENIZERS_PARALLELISM: "false",
-        PYTHONNOUSERSITE: "1",
-        OMP_NUM_THREADS: threads,
-        MKL_NUM_THREADS: threads,
-        OPENBLAS_NUM_THREADS: threads,
-        HTTP_PROXY: "",
-        HTTPS_PROXY: "",
-        ALL_PROXY: "",
-        NO_PROXY: "*",
-      },
-    });
-    this.child = child;
-    child.stdout.setEncoding("utf8");
-    child.stdout.on("data", (chunk: string) => this.receive(chunk));
-    child.stderr.resume();
-    child.once("error", () => this.failWorker());
-    child.once("close", () => this.failWorker());
-    const startupTimer = setTimeout(() => this.failWorker(), this.options.startupTimeoutMs ?? 120_000);
-    startupTimer.unref();
-    try {
-      await this.ready;
-    } finally {
-      clearTimeout(startupTimer);
-    }
-  }
-
-  private async ensureStartedWithin(signal: AbortSignal, deadline: number): Promise<void> {
-    const started = this.ensureStarted();
-    await new Promise<void>((resolve, reject) => {
-      let settled = false;
-      const finish = (error?: Error, stopWorker = false) => {
-        if (settled) return;
-        settled = true;
-        clearTimeout(timer);
-        signal.removeEventListener("abort", cancel);
-        if (stopWorker) this.failWorker();
-        if (error) reject(error);
-        else resolve();
-      };
-      const cancel = () => finish(new LocalNerUnavailableError(), true);
-      const timer = setTimeout(cancel, Math.max(1, Math.floor(deadline - Date.now())));
-      signal.addEventListener("abort", cancel, { once: true });
-      void started.then(
-        () => finish(),
-        () => finish(new LocalNerUnavailableError()),
-      );
-    });
-  }
-
-  private receive(chunk: string): void {
-    this.stdout += chunk;
-    if (Buffer.byteLength(this.stdout, "utf8") > MAX_LINE_BYTES) {
-      this.failWorker();
-      return;
-    }
-    let newline: number;
-    while ((newline = this.stdout.indexOf("\n")) >= 0) {
-      const line = this.stdout.slice(0, newline);
-      this.stdout = this.stdout.slice(newline + 1);
-      if (!line) continue;
-      try {
-        const message = workerMessageSchema.parse(JSON.parse(line));
-        if ("ready" in message) {
-          this.workerReady = true;
-          this.readyResolve?.();
-          this.readyResolve = undefined;
-          this.readyReject = undefined;
-          continue;
-        }
-        const pending = this.pending.get(message.id);
-        if (!pending) continue;
-        this.finishPending(message.id);
-        if (message.ok) pending.resolve(message.evidence);
-        else pending.reject(new LocalNerUnavailableError());
-      } catch {
-        this.failWorker();
-        return;
-      }
-    }
-  }
-
-  private finishPending(id: string): void {
-    const pending = this.pending.get(id);
-    if (!pending) return;
-    clearTimeout(pending.timer);
-    pending.signal.removeEventListener("abort", pending.cancel);
-    this.pending.delete(id);
-  }
-
-  private stopWorker(): void {
-    const child = this.child;
-    if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGTERM");
-  }
-
-  private failWorker(): void {
-    const error = new LocalNerUnavailableError();
-    this.readyReject?.(error);
-    this.readyResolve = undefined;
-    this.readyReject = undefined;
-    for (const [id, pending] of this.pending) {
-      this.finishPending(id);
-      pending.reject(error);
-    }
-    this.stopWorker();
-    this.child = undefined;
-    this.ready = undefined;
-    this.workerReady = false;
-    this.stdout = "";
-  }
-}

backend/src/catalog/memory-repository.ts

@@ -18,8 +18,6 @@ import {
   type CatalogLogicalRelationshipCandidate,
   type CatalogLogicalRelationshipContext,
   type CatalogPhysicalRelationship,
-  type CatalogPreprocessingStartResult,
-  type CatalogPreprocessingClearResult,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -37,10 +35,10 @@ import {
   type DescriptionGenerationRun,
   type DescriptionGenerationRunUpdate,
   type DescriptionGenerationScope,
-  type SensitivityAnalysisEvent,
-  type SensitivityAnalysisRun,
-  type SensitivityAnalysisRunUpdate,
-  type SensitivityAnalysisScope,
+  type SensitiveDataSuggestionEvent,
+  type SensitiveDataSuggestionRun,
+  type SensitiveDataSuggestionRunUpdate,
+  type SensitiveDataSuggestionScope,
   type TableSyncRepositoryResult,
   type WorkspaceDatabase,
 } from "./types.js";
@@ -58,8 +56,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
   private readonly logicalRelationships = new Map<string, CatalogLogicalRelationship>();
   private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
   private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
-  private readonly sensitivityAnalysisRuns = new Map<string, SensitivityAnalysisRun>();
-  private readonly sensitivityAnalysisEvents = new Map<string, SensitivityAnalysisEvent[]>();
+  private readonly sensitiveDataSuggestionRuns = new Map<string, SensitiveDataSuggestionRun>();
+  private readonly sensitiveDataSuggestionEvents = new Map<string, SensitiveDataSuggestionEvent[]>();
   private readonly syncRuns = new Map<string, CatalogSyncRun>();
   private readonly syncEvents = new Map<string, CatalogSyncEvent[]>();
 
@@ -74,77 +72,6 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
     return value ? clone(value) : undefined;
   }
-  async beginPreprocessing(
-    workspaceId: string,
-    inputFingerprint: string,
-  ): Promise<CatalogPreprocessingStartResult> {
-    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
-    if (!database) return { kind: "not_found" };
-    if (database.preprocessingStatus === "running") return { kind: "already_running" };
-    if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
-    const catalogBusy = [...this.syncRuns.values()].some((run) =>
-      run.databaseId === database.id
-      && ["queued", "running", "awaiting_confirmation", "applying"].includes(run.state))
-      || [...this.descriptionGenerationRuns.values()].some((run) =>
-        run.databaseId === database.id && ["queued", "running"].includes(run.status))
-      || [...this.sensitivityAnalysisRuns.values()].some((run) =>
-        run.databaseId === database.id && ["queued", "running"].includes(run.status));
-    if (catalogBusy) return { kind: "catalog_busy" };
-    const now = new Date().toISOString();
-    const updated: WorkspaceDatabase = {
-      ...database,
-      preprocessingStatus: "running",
-      preprocessingInputFingerprint: inputFingerprint,
-      preprocessedMetadataRevision: undefined,
-      preprocessingStartedAt: now,
-      preprocessingFinishedAt: undefined,
-      preprocessingErrorCode: undefined,
-      updatedAt: now,
-    };
-    this.records.set(database.id, updated);
-    return { kind: "started", database: clone(updated) };
-  }
-  async finishPreprocessing(
-    workspaceId: string,
-    metadataContentRevision: number,
-    inputFingerprint: string,
-    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
-  ): Promise<WorkspaceDatabase | undefined> {
-    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
-    if (!database
-      || database.preprocessingStatus !== "running"
-      || database.metadataContentRevision !== metadataContentRevision
-      || database.preprocessingInputFingerprint !== inputFingerprint) return undefined;
-    const updated: WorkspaceDatabase = {
-      ...database,
-      preprocessingStatus: outcome.status,
-      preprocessedMetadataRevision: outcome.status === "succeeded"
-        ? metadataContentRevision
-        : undefined,
-      preprocessingFinishedAt: new Date().toISOString(),
-      preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : undefined,
-    };
-    this.records.set(database.id, updated);
-    return clone(updated);
-  }
-  async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
-    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
-    if (!database) return { kind: "not_found" };
-    if (database.preprocessingStatus === "running") return { kind: "already_running" };
-    const now = new Date().toISOString();
-    const updated: WorkspaceDatabase = {
-      ...database,
-      preprocessingStatus: "failed",
-      preprocessingInputFingerprint: undefined,
-      preprocessedMetadataRevision: undefined,
-      preprocessingStartedAt: undefined,
-      preprocessingFinishedAt: now,
-      preprocessingErrorCode: "derived_data_cleared",
-      updatedAt: now,
-    };
-    this.records.set(database.id, updated);
-    return { kind: "cleared", database: clone(updated) };
-  }
   async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
     if (databaseId !== undefined && !this.records.has(databaseId)) return undefined;
 
@@ -156,10 +83,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const tableIds = new Set(tables.map((table) => table.id));
     const columns = [...this.columns.values()]
       .filter((column) => tableIds.has(column.tableId));
-    const relationships = [
-      ...this.relationships.values(),
-      ...this.logicalRelationships.values(),
-    ].filter((relationship) => selectedDatabaseIds.has(relationship.databaseId));
+    const relationships = [...this.relationships.values()]
+      .filter((relationship) => selectedDatabaseIds.has(relationship.databaseId));
 
     return createCatalogMetrics(databaseId, {
       tables: tables.length,
@@ -187,8 +112,6 @@ export class MemoryCatalogRepository implements CatalogRepository {
       createdAt: now,
       updatedAt: now,
       connectionStatus: "untested",
-      metadataContentRevision: 0,
-      preprocessingStatus: "failed",
     };
     this.records.set(record.id, record);
     return clone(record);
@@ -260,10 +183,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
       this.descriptionGenerationRuns.delete(runId);
       this.descriptionGenerationEvents.delete(runId);
     }
-    for (const [runId, run] of this.sensitivityAnalysisRuns) {
+    for (const [runId, run] of this.sensitiveDataSuggestionRuns) {
       if (run.databaseId !== id) continue;
-      this.sensitivityAnalysisRuns.delete(runId);
-      this.sensitivityAnalysisEvents.delete(runId);
+      this.sensitiveDataSuggestionRuns.delete(runId);
+      this.sensitiveDataSuggestionEvents.delete(runId);
     }
     return this.records.delete(id);
   }
@@ -332,7 +255,6 @@ export class MemoryCatalogRepository implements CatalogRepository {
     description: string | null,
     generatedDescription: string | null,
     sensitive?: boolean,
-    sensitivityReason?: string | null,
   ): Promise<CatalogColumn | undefined> {
     const current = await this.getColumn(databaseId, tableId, columnId);
     if (!current || current.version !== expectedVersion) return undefined;
@@ -341,9 +263,6 @@ export class MemoryCatalogRepository implements CatalogRepository {
       description,
       generatedDescription,
       sensitive: sensitive ?? current.sensitive,
-      sensitivityReason: sensitive === false
-        ? null
-        : sensitivityReason === undefined ? current.sensitivityReason : sensitivityReason,
       version: current.version + 1,
       updatedAt: new Date().toISOString(),
     };
@@ -357,39 +276,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
     targetIds: readonly string[],
   ): Promise<CatalogDescriptionConsolidationCounts | undefined> {
     const selectedTargetIds = [...new Set(targetIds)];
-    if (!this.records.has(databaseId)) {
+    if (!this.records.has(databaseId) || selectedTargetIds.length === 0) {
       return undefined;
     }
     const now = new Date().toISOString();
-    if (target === "database" || target === "database_columns") {
-      const tableTargets = target === "database"
-        ? [...this.tables.values()].filter((table) => table.databaseId === databaseId)
-        : [];
-      const columnTargets = [...this.columns.values()].filter((column) => (
-        this.tables.get(column.tableId)?.databaseId === databaseId
-      ));
-      const copiedTables = tableTargets.filter((table) => Boolean(table.generatedDescription?.trim()));
-      const copiedColumns = columnTargets.filter((column) => Boolean(column.generatedDescription?.trim()));
-      for (const table of copiedTables) {
-        this.tables.set(table.id, {
-          ...table,
-          description: table.generatedDescription,
-          version: table.version + 1,
-          updatedAt: now,
-        });
-      }
-      for (const column of copiedColumns) {
-        this.columns.set(column.id, {
-          ...column,
-          description: column.generatedDescription,
-          version: column.version + 1,
-          updatedAt: now,
-        });
-      }
-      const copied = copiedTables.length + copiedColumns.length;
-      return { copied, skipped: tableTargets.length + columnTargets.length - copied };
-    }
-    if (selectedTargetIds.length === 0) return undefined;
     if (target === "tables") {
       const targets = selectedTargetIds.map((id) => this.tables.get(id));
       if (targets.some((table) => !table || table.databaseId !== databaseId)) return undefined;
@@ -543,96 +433,93 @@ export class MemoryCatalogRepository implements CatalogRepository {
       .map((event) => structuredClone(event));
   }
 
-  async createSensitivityAnalysisRun(
+  async createSensitiveDataSuggestionRun(
     databaseId: string,
-    scope: SensitivityAnalysisScope,
-    origin: { engine: "llm"; modelId: string } | { engine: "local"; policyVersion: string },
-  ): Promise<SensitivityAnalysisRun> {
+    scope: SensitiveDataSuggestionScope,
+    modelId: string,
+  ): Promise<SensitiveDataSuggestionRun> {
     const now = new Date().toISOString();
-    const run: SensitivityAnalysisRun = {
+    const run: SensitiveDataSuggestionRun = {
       id: randomUUID(),
       databaseId,
       scope,
-      engine: origin.engine,
-      modelId: origin.engine === "llm" ? origin.modelId : null,
-      policyVersion: origin.engine === "local" ? origin.policyVersion : null,
+      modelId,
       status: "running",
       total: 0,
       suggestedSensitive: 0,
-      suggestedNonSensitive: 0,
-      unknown: 0,
-      inputTokens: 0,
-      cacheReadTokens: 0,
-      outputTokens: 0,
+        suggestedNonSensitive: 0,
+        inputTokens: 0,
+        cacheReadTokens: 0,
+        outputTokens: 0,
       createdAt: now,
       startedAt: now,
       updatedAt: now,
       finishedAt: null,
       errorSummary: null,
     };
-    this.sensitivityAnalysisRuns.set(run.id, run);
+    this.sensitiveDataSuggestionRuns.set(run.id, run);
     return structuredClone(run);
   }
 
-  async getSensitivityAnalysisRun(
+  async getSensitiveDataSuggestionRun(
     runId: string,
-  ): Promise<SensitivityAnalysisRun | undefined> {
-    const run = this.sensitivityAnalysisRuns.get(runId);
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
+    const run = this.sensitiveDataSuggestionRuns.get(runId);
     return run ? structuredClone(run) : undefined;
   }
 
-  async listSensitivityAnalysisRuns(limit = 50): Promise<SensitivityAnalysisRun[]> {
-    return [...this.sensitivityAnalysisRuns.values()]
+  async listSensitiveDataSuggestionRuns(limit = 50): Promise<SensitiveDataSuggestionRun[]> {
+    return [...this.sensitiveDataSuggestionRuns.values()]
       .sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id))
       .slice(0, limit)
       .map((run) => structuredClone(run));
   }
 
-  async interruptActiveSensitivityAnalysisRuns(
+  async interruptActiveSensitiveDataSuggestionRuns(
     errorSummary: string,
-  ): Promise<SensitivityAnalysisRun[]> {
-    const interrupted: SensitivityAnalysisRun[] = [];
-    for (const run of this.sensitivityAnalysisRuns.values()) {
+  ): Promise<SensitiveDataSuggestionRun[]> {
+    const interrupted: SensitiveDataSuggestionRun[] = [];
+    for (const run of this.sensitiveDataSuggestionRuns.values()) {
       if (run.status !== "running") continue;
       const now = new Date().toISOString();
-      const updated: SensitivityAnalysisRun = {
+      const updated: SensitiveDataSuggestionRun = {
         ...run,
         status: "interrupted",
         updatedAt: now,
         finishedAt: now,
         errorSummary,
       };
-      this.sensitivityAnalysisRuns.set(run.id, updated);
+      this.sensitiveDataSuggestionRuns.set(run.id, updated);
       interrupted.push(structuredClone(updated));
     }
     return interrupted;
   }
 
-  async updateSensitivityAnalysisRun(
+  async updateSensitiveDataSuggestionRun(
     runId: string,
-    update: SensitivityAnalysisRunUpdate,
-  ): Promise<SensitivityAnalysisRun | undefined> {
-    const current = this.sensitivityAnalysisRuns.get(runId);
+    update: SensitiveDataSuggestionRunUpdate,
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
+    const current = this.sensitiveDataSuggestionRuns.get(runId);
     if (!current) return undefined;
     const updated = {
       ...current,
       ...structuredClone(update),
       updatedAt: new Date().toISOString(),
     };
-    this.sensitivityAnalysisRuns.set(runId, updated);
+    this.sensitiveDataSuggestionRuns.set(runId, updated);
     return structuredClone(updated);
   }
 
-  async appendSensitivityAnalysisEvent(
+  async appendSensitiveDataSuggestionEvent(
     runId: string,
-    level: SensitivityAnalysisEvent["level"],
+    level: SensitiveDataSuggestionEvent["level"],
     message: string,
-  ): Promise<SensitivityAnalysisEvent> {
-    if (!this.sensitivityAnalysisRuns.has(runId)) {
-      throw new CatalogConflictError("Sensitivity Analysis Run does not exist");
+  ): Promise<SensitiveDataSuggestionEvent> {
+    if (!this.sensitiveDataSuggestionRuns.has(runId)) {
+      throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist");
     }
-    const events = this.sensitivityAnalysisEvents.get(runId) ?? [];
-    const event: SensitivityAnalysisEvent = {
+    const events = this.sensitiveDataSuggestionEvents.get(runId) ?? [];
+    const event: SensitiveDataSuggestionEvent = {
       runId,
       sequence: events.length + 1,
       level,
@@ -640,15 +527,15 @@ export class MemoryCatalogRepository implements CatalogRepository {
       createdAt: new Date().toISOString(),
     };
     events.push(event);
-    this.sensitivityAnalysisEvents.set(runId, events);
+    this.sensitiveDataSuggestionEvents.set(runId, events);
     return structuredClone(event);
   }
 
-  async listSensitivityAnalysisEvents(
+  async listSensitiveDataSuggestionEvents(
     runId: string,
     afterSequence = 0,
-  ): Promise<SensitivityAnalysisEvent[]> {
-    return (this.sensitivityAnalysisEvents.get(runId) ?? [])
+  ): Promise<SensitiveDataSuggestionEvent[]> {
+    return (this.sensitiveDataSuggestionEvents.get(runId) ?? [])
       .filter((event) => event.sequence > afterSequence)
       .map((event) => structuredClone(event));
   }
@@ -797,22 +684,18 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const tables = [...this.tables.values()].filter((table) => selected.has(table.databaseId));
     const tableIds = new Set(tables.map((table) => table.id));
     const columns = [...this.columns.values()].filter((column) => tableIds.has(column.tableId));
-    const physicalRelationships = [...this.relationships.values()]
+    const relationships = [...this.relationships.values()]
       .filter((relationship) => selected.has(relationship.databaseId));
-    const logicalRelationships = [...this.logicalRelationships.values()]
-      .filter((relationship) => selected.has(relationship.databaseId));
-    const relationshipCount = physicalRelationships.length + logicalRelationships.length;
 
     if (target === "tables") {
       for (const table of tables) this.deleteTable(table.id);
       this.markCatalogIncomplete(selectedDatabaseIds);
-      return { tables: tables.length, columns: columns.length, relationships: relationshipCount };
+      return { tables: tables.length, columns: columns.length, relationships: relationships.length };
     }
-    for (const relationship of physicalRelationships) this.relationships.delete(relationship.id);
-    for (const relationship of logicalRelationships) this.logicalRelationships.delete(relationship.id);
+    for (const relationship of relationships) this.relationships.delete(relationship.id);
     for (const databaseId of selectedDatabaseIds) this.refreshForeignKeyFlags(databaseId);
     this.markCatalogIncomplete(selectedDatabaseIds);
-    return { tables: 0, columns: 0, relationships: relationshipCount };
+    return { tables: 0, columns: 0, relationships: relationships.length };
   }
 
   async deleteTableMetadata(
@@ -850,23 +733,14 @@ export class MemoryCatalogRepository implements CatalogRepository {
       return { tables: 0, columns: columns.length, relationships: 0 };
     }
 
-    const physicalRelationships = [...this.relationships.values()].filter((relationship) => (
+    const relationships = [...this.relationships.values()].filter((relationship) => (
       relationship.databaseId === databaseId
       && (selected.has(relationship.sourceTableId) || selected.has(relationship.targetTableId))
     ));
-    const logicalRelationships = [...this.logicalRelationships.values()].filter((relationship) => (
-      relationship.databaseId === databaseId
-      && (selected.has(relationship.sourceTableId) || selected.has(relationship.targetTableId))
-    ));
-    for (const relationship of physicalRelationships) this.relationships.delete(relationship.id);
-    for (const relationship of logicalRelationships) this.logicalRelationships.delete(relationship.id);
+    for (const relationship of relationships) this.relationships.delete(relationship.id);
     this.refreshForeignKeyFlags(databaseId);
     this.markCatalogIncomplete([databaseId]);
-    return {
-      tables: 0,
-      columns: 0,
-      relationships: physicalRelationships.length + logicalRelationships.length,
-    };
+    return { tables: 0, columns: 0, relationships: relationships.length };
   }
 
   async planSchemaSync(
@@ -1050,7 +924,6 @@ export class MemoryCatalogRepository implements CatalogRepository {
             description: null,
             generatedDescription: null,
             sensitive: false,
-            sensitivityReason: null,
             lastSyncedDatabaseVersion: expectedDatabaseVersion,
             lastSyncedAt: now,
             version: 1,

backend/src/catalog/metadata-generation-models.ts

@@ -1,5 +1,63 @@
-import { loadSecretBundle } from "../config/secret-bundle.js";
-import { loadRuntimeModelCatalog } from "../models/runtime-model-catalog.js";
+import {
+  closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
+  type Stats,
+} from "node:fs";
+import { parseAllDocuments } from "yaml";
+import { z } from "zod";
+import {
+  loadSecretBundle,
+  METADATA_GENERATION_SECRET_KEYS,
+} from "../config/secret-bundle.js";
+
+const MAX_INSTALLATION_BYTES = 1024 * 1024;
+const RUNTIME_INSTALLATION_FILE = "/run/thothii-installation/thothii-installation.yaml";
+const modelId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
+const apiKeyEnvironment = z.enum(METADATA_GENERATION_SECRET_KEYS);
+const endpointSchema = z.object({
+  baseUrl: z.string().min(1).max(2048).refine((value) => {
+    try {
+      const url = new URL(value);
+      return (url.protocol === "http:" || url.protocol === "https:")
+        && url.username === "" && url.password === "" && url.search === "" && url.hash === "";
+    } catch {
+      return false;
+    }
+  }),
+  apiVersion: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/).optional(),
+}).strict();
+const configuredModelSchema = z.object({
+  id: modelId,
+  label: z.string().min(1).max(128).refine((value) => value.trim() === value && !/\p{Cc}/u.test(value)),
+  litellm: z.object({
+    provider: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/),
+    model: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/),
+    disableThinking: z.literal(true).optional(),
+    endpoint: endpointSchema.optional(),
+  }).strict(),
+  apiKeyEnv: apiKeyEnvironment.optional(),
+}).strict().superRefine((value, context) => {
+  if (value.apiKeyEnv === undefined && value.litellm.endpoint === undefined) {
+    context.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ["apiKeyEnv"],
+      message: "keyless models require an explicit endpoint",
+    });
+  }
+  if (value.litellm.disableThinking === true && value.litellm.endpoint === undefined) {
+    context.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ["litellm", "disableThinking"],
+      message: "thinking may be disabled only for an explicit endpoint",
+    });
+  }
+});
+const metadataGenerationSchema = z.object({
+  default: modelId.optional(),
+  models: z.array(configuredModelSchema).max(64).default([]),
+}).strict();
+const installationSchema = z.object({
+  metadataGeneration: metadataGenerationSchema.optional(),
+}).passthrough();
 
 export interface MetadataGenerationModelChoice {
   id: string;
@@ -28,6 +86,7 @@ export class MetadataGenerationModelUnavailableError extends Error {
   }
 }
 
+/** The complete interface callers need: safe discovery plus fail-closed runtime resolution. */
 export interface MetadataGenerationModels {
   catalog(): MetadataGenerationModelCatalog;
   resolve(selection: string): ResolvedMetadataGenerationModel;
@@ -37,78 +96,140 @@ class RestartLoadedMetadataGenerationModels implements MetadataGenerationModels
   readonly #models: ReadonlyMap<string, ResolvedMetadataGenerationModel>;
   readonly #catalog: MetadataGenerationModelCatalog;
 
-  constructor(models: ReadonlyMap<string, ResolvedMetadataGenerationModel>, defaultModel: string | null) {
+  constructor(
+    models: ReadonlyMap<string, ResolvedMetadataGenerationModel> = new Map(),
+    defaultModel: string | null = null,
+    choices: MetadataGenerationModelChoice[] = [],
+  ) {
     this.#models = models;
     this.#catalog = {
-      models: [...models.values()].map(({ id }) => ({ id, label: id })),
+      models: choices.map((choice) => ({ ...choice })),
       default: defaultModel,
     };
   }
 
   catalog(): MetadataGenerationModelCatalog {
-    return { models: this.#catalog.models.map((choice) => ({ ...choice })), default: this.#catalog.default };
+    return {
+      models: this.#catalog.models.map((choice) => ({ ...choice })),
+      default: this.#catalog.default,
+    };
   }
 
   resolve(selection: string): ResolvedMetadataGenerationModel {
-    const model = this.#models.get(selection);
+    const model = typeof selection === "string" ? this.#models.get(selection) : undefined;
     if (!model) throw new MetadataGenerationModelUnavailableError();
     return model;
   }
 }
 
-function invalid(message = "metadata-generation runtime catalog is invalid"): Error {
+function invalid(message = "metadata-generation configuration is invalid"): Error {
   return new Error(message);
 }
 
+function protectedInstallationStat(file: string, info: Stats): boolean {
+  const mode = info.mode & 0o777;
+  if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
+    || info.size < 1 || info.size > MAX_INSTALLATION_BYTES) return false;
+  if (file === RUNTIME_INSTALLATION_FILE && info.uid === 0 && mode === 0o444) return true;
+  return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600);
+}
+
+function readProtectedInstallation(file: string): string {
+  let descriptor: number | undefined;
+  try {
+    const before = lstatSync(file);
+    if (!protectedInstallationStat(file, before)) throw new Error("unavailable");
+    descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
+    const opened = fstatSync(descriptor);
+    if (!protectedInstallationStat(file, opened)
+      || before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("unavailable");
+    const source = readFileSync(descriptor, "utf8");
+    const after = fstatSync(descriptor);
+    const current = lstatSync(file);
+    if (!protectedInstallationStat(file, after) || !protectedInstallationStat(file, current)
+      || opened.dev !== after.dev || opened.ino !== after.ino
+      || opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("unavailable");
+    return source;
+  } finally {
+    if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized below */ }
+  }
+}
+
+function readInstallation(file: string): unknown {
+  try {
+    const documents = parseAllDocuments(readProtectedInstallation(file), { uniqueKeys: true });
+    if (documents.length !== 1) throw invalid("metadata-generation installation must contain one YAML document");
+    const document = documents[0];
+    if (document.errors.length > 0 || document.warnings.length > 0) {
+      throw invalid("metadata-generation installation contains invalid YAML");
+    }
+    return document.toJSON();
+  } catch (error) {
+    if (error instanceof Error && error.message.startsWith("metadata-generation")) throw error;
+    throw invalid("metadata-generation installation is unavailable");
+  }
+}
+
 export function loadMetadataGenerationModels(options: {
-  catalogFile?: string;
+  installationFile?: string;
   secretsFile?: string;
 }): MetadataGenerationModels {
-  const catalog = loadRuntimeModelCatalog(options.catalogFile);
-  const configured = catalog.metadataModels();
-  if (configured.length === 0) return new RestartLoadedMetadataGenerationModels(new Map(), null);
+  if (!options.installationFile) return new RestartLoadedMetadataGenerationModels();
+  const installation = installationSchema.safeParse(readInstallation(options.installationFile));
+  if (!installation.success) throw invalid();
+  const configured = installation.data.metadataGeneration;
+  if (!configured || configured.models.length === 0) {
+    if (configured?.default !== undefined) throw invalid("metadata-generation default does not identify a configured model");
+    return new RestartLoadedMetadataGenerationModels();
+  }
+  if (!configured.default) throw invalid("metadata-generation default is required when models are configured");
 
-  const requiresSecrets = configured.some((model) => model.authentication.mode === "secret_env");
+  const seen = new Set<string>();
+  for (const model of configured.models) {
+    if (seen.has(model.id)) throw invalid(`metadata-generation model id "${model.id}" is duplicated`);
+    seen.add(model.id);
+  }
+  if (!seen.has(configured.default)) {
+    throw invalid(`metadata-generation default "${configured.default}" is not configured`);
+  }
+  const requiresSecrets = configured.models.some((model) => model.apiKeyEnv !== undefined);
   let secrets: ReadonlyMap<string, string> = new Map();
   if (requiresSecrets) {
     if (!options.secretsFile) throw invalid("metadata-generation keyed models require THT_SECRETS_FILE");
-    try { secrets = loadSecretBundle(options.secretsFile); }
-    catch { throw invalid("metadata-generation secrets are unavailable"); }
+    try {
+      secrets = loadSecretBundle(options.secretsFile);
+    } catch {
+      throw invalid("metadata-generation secrets are unavailable");
+    }
   }
-
   const models = new Map<string, ResolvedMetadataGenerationModel>();
-  const labels = new Map<string, string>();
-  for (const configuredModel of configured) {
-    const adapter = configuredModel.metadataAdapter;
-    if (!adapter || configuredModel.authentication.mode === "pi_auth") throw invalid();
-    const apiKeyEnv = configuredModel.authentication.apiKeyEnv;
+  for (const configuredModel of configured.models) {
     let apiKey: string | undefined;
-    if (configuredModel.authentication.mode === "secret_env") {
-      if (!apiKeyEnv) throw invalid();
-      apiKey = secrets.get(apiKeyEnv);
-      if (!apiKey) throw invalid(`metadata-generation model "${configuredModel.id}" secret "${apiKeyEnv}" is missing`);
+    if (configuredModel.apiKeyEnv !== undefined) {
+      apiKey = secrets.get(configuredModel.apiKeyEnv);
+      if (!apiKey) {
+        throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is missing`);
+      }
       if (apiKey.length > 16 * 1024 || /\s/u.test(apiKey)) {
-        throw invalid(`metadata-generation model "${configuredModel.id}" secret "${apiKeyEnv}" is unusable`);
+        throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is unusable`);
       }
     }
-    labels.set(configuredModel.id, configuredModel.label);
     models.set(configuredModel.id, Object.freeze({
       id: configuredModel.id,
-      provider: adapter.litellmProvider,
-      model: configuredModel.upstreamModel,
-      ...(configuredModel.metadataGeneration?.disableThinking === true
-        ? { disableThinking: true as const } : {}),
-      ...(configuredModel.endpoint ? { endpoint: Object.freeze({ ...configuredModel.endpoint }) } : {}),
-      ...(apiKeyEnv ? { apiKeyEnv, apiKey } : {}),
+      provider: configuredModel.litellm.provider,
+      model: configuredModel.litellm.model,
+      ...(configuredModel.litellm.disableThinking === true ? { disableThinking: true as const } : {}),
+      ...(configuredModel.litellm.endpoint === undefined
+        ? {}
+        : { endpoint: Object.freeze({ ...configuredModel.litellm.endpoint }) }),
+      ...(configuredModel.apiKeyEnv === undefined
+        ? {}
+        : { apiKeyEnv: configuredModel.apiKeyEnv, apiKey }),
     }));
   }
-  const result = new RestartLoadedMetadataGenerationModels(models, catalog.defaultMetadataGeneration);
-  const safe = result.catalog();
-  return {
-    catalog: () => ({
-      default: safe.default,
-      models: safe.models.map((choice) => ({ ...choice, label: labels.get(choice.id) ?? choice.id })),
-    }),
-    resolve: (selection) => result.resolve(selection),
-  };
+  return new RestartLoadedMetadataGenerationModels(
+    models,
+    configured.default,
+    configured.models.map(({ id, label }) => ({ id, label })),
+  );
 }

backend/src/catalog/metadata-snapshot.ts

@@ -1,144 +0,0 @@
-import type {
-  CatalogColumn,
-  CatalogLogicalRelationship,
-  CatalogPhysicalRelationship,
-  CatalogRepository,
-  CatalogTable,
-} from "./types.js";
-
-export type CatalogDescriptionSource = "curated" | "generated" | "source_comment";
-
-export interface CatalogMetadataSnapshotColumn {
-  id: string;
-  name: string;
-  ordinalPosition: number;
-  dataType: string;
-  isNullable: boolean;
-  defaultExpression: string | null;
-  primaryKeyPosition: number | null;
-  sensitive: boolean;
-  description: string | null;
-  descriptionSource: CatalogDescriptionSource | null;
-}
-
-export interface CatalogMetadataSnapshotTable {
-  id: string;
-  name: string;
-  description: string | null;
-  descriptionSource: CatalogDescriptionSource | null;
-  columns: CatalogMetadataSnapshotColumn[];
-}
-
-export interface CatalogMetadataSnapshotRelationship {
-  id: string;
-  origin: "physical" | "generated" | "manual";
-  sourceTable: string;
-  sourceColumns: string[];
-  targetTable: string;
-  targetColumns: string[];
-}
-
-export interface CatalogMetadataSnapshot {
-  schemaVersion: 1;
-  workspaceId: string;
-  databaseId: string;
-  databaseName: string;
-  schemaName: string;
-  metadataContentRevision: number;
-  tables: CatalogMetadataSnapshotTable[];
-  relationships: CatalogMetadataSnapshotRelationship[];
-}
-
-function effectiveDescription(value: {
-  description: string | null;
-  generatedDescription: string | null;
-  sourceComment: string | null;
-}): { description: string | null; descriptionSource: CatalogDescriptionSource | null } {
-  if (value.description?.trim()) {
-    return { description: value.description.trim(), descriptionSource: "curated" };
-  }
-  if (value.generatedDescription?.trim()) {
-    return { description: value.generatedDescription.trim(), descriptionSource: "generated" };
-  }
-  if (value.sourceComment?.trim()) {
-    return { description: value.sourceComment.trim(), descriptionSource: "source_comment" };
-  }
-  return { description: null, descriptionSource: null };
-}
-
-function snapshotColumn(column: CatalogColumn): CatalogMetadataSnapshotColumn {
-  return {
-    id: column.id,
-    name: column.name,
-    ordinalPosition: column.ordinalPosition,
-    dataType: column.dataType,
-    isNullable: column.isNullable,
-    defaultExpression: column.defaultExpression,
-    primaryKeyPosition: column.primaryKeyPosition,
-    sensitive: column.sensitive,
-    ...effectiveDescription(column),
-  };
-}
-
-function snapshotRelationship(
-  relationship: CatalogPhysicalRelationship | CatalogLogicalRelationship,
-): CatalogMetadataSnapshotRelationship {
-  const columns = [...relationship.columns].sort((left, right) => left.position - right.position);
-  return {
-    id: relationship.id,
-    origin: relationship.origin,
-    sourceTable: relationship.sourceTableName,
-    sourceColumns: columns.map((column) => column.sourceColumnName),
-    targetTable: relationship.targetTableName,
-    targetColumns: columns.map((column) => column.targetColumnName),
-  };
-}
-
-export async function buildCatalogMetadataSnapshot(
-  repository: CatalogRepository,
-  workspaceId: string,
-  expectedMetadataContentRevision: number,
-): Promise<CatalogMetadataSnapshot> {
-  const database = await repository.getByWorkspace(workspaceId);
-  if (!database || database.preprocessingStatus !== "running") {
-    throw new Error("catalog preprocessing lease is not active");
-  }
-  if (database.metadataContentRevision !== expectedMetadataContentRevision) {
-    throw new Error("catalog metadata revision changed");
-  }
-
-  const catalogTables = await repository.listTables(database.id);
-  const tables: CatalogMetadataSnapshotTable[] = [];
-  for (const table of [...catalogTables].sort((left, right) => left.name.localeCompare(right.name))) {
-    const columns = await repository.listColumns(database.id, table.id);
-    tables.push({
-      id: table.id,
-      name: table.name,
-      ...effectiveDescription(table),
-      columns: columns
-        .sort((left, right) => left.ordinalPosition - right.ordinalPosition || left.name.localeCompare(right.name))
-        .map(snapshotColumn),
-    });
-  }
-
-  const physical = await repository.listRelationships(database.id);
-  const logical = (await repository.listLogicalRelationships(database.id))
-    .filter((relationship) => relationship.status === "active");
-  const relationships = [...physical, ...logical]
-    .map(snapshotRelationship)
-    .sort((left, right) =>
-      left.sourceTable.localeCompare(right.sourceTable)
-      || left.targetTable.localeCompare(right.targetTable)
-      || left.id.localeCompare(right.id));
-
-  return {
-    schemaVersion: 1,
-    workspaceId,
-    databaseId: database.id,
-    databaseName: database.databaseName,
-    schemaName: database.schema,
-    metadataContentRevision: expectedMetadataContentRevision,
-    tables,
-    relationships,
-  };
-}

backend/src/catalog/migrate.ts

@@ -9,13 +9,9 @@ import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_syn
 import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js";
 import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
 import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
-import * as sensitivityAnalysisRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
+import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
 import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
 import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
-import * as canonicalModelIdsMigration from "./migrations/010_canonical_model_ids.js";
-import * as localSensitivityAnalysisMigration from "./migrations/011_local_sensitivity_analysis.js";
-import * as sensitivityReasonMigration from "./migrations/012_sensitivity_reason.js";
-import * as catalogPreprocessingStateMigration from "./migrations/013_catalog_preprocessing_state.js";
 
 const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
 const host = process.env.THT_CATALOG_DB_HOST;
@@ -47,13 +43,9 @@ const provider: MigrationProvider = {
       "004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration,
       "005_description_generation_runs": descriptionGenerationRunsMigration,
       "006_sensitive_data_flag": sensitiveDataFlagMigration,
-      "007_sensitive_data_suggestion_runs": sensitivityAnalysisRunsMigration,
+      "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
       "008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
       "009_ai_token_usage": aiTokenUsageMigration,
-      "010_canonical_model_ids": canonicalModelIdsMigration,
-      "011_local_sensitivity_analysis": localSensitivityAnalysisMigration,
-      "012_sensitivity_reason": sensitivityReasonMigration,
-      "013_catalog_preprocessing_state": catalogPreprocessingStateMigration,
     };
   },
 };

backend/src/catalog/migrations/010_canonical_model_ids.ts

@@ -1,28 +0,0 @@
-import { sql, type Kysely } from "kysely";
-import type { CatalogDatabase } from "../repository.js";
-
-const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$";
-const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$";
-const historicalOrCanonicalModelPattern = `(${legacyModelPattern})|(${canonicalModelPattern})`;
-
-export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
-  await sql.raw(`alter table description_generation_runs
-    drop constraint description_generation_runs_model_id_check,
-    add constraint description_generation_runs_model_id_check
-      check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
-  await sql.raw(`alter table sensitive_data_suggestion_runs
-    drop constraint sensitive_data_suggestion_runs_model_id_check,
-    add constraint sensitive_data_suggestion_runs_model_id_check
-      check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
-}
-
-export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
-  await sql.raw(`alter table sensitive_data_suggestion_runs
-    drop constraint sensitive_data_suggestion_runs_model_id_check,
-    add constraint sensitive_data_suggestion_runs_model_id_check
-      check (model_id ~ '${legacyModelPattern}')`).execute(db);
-  await sql.raw(`alter table description_generation_runs
-    drop constraint description_generation_runs_model_id_check,
-    add constraint description_generation_runs_model_id_check
-      check (model_id ~ '${legacyModelPattern}')`).execute(db);
-}

backend/src/catalog/migrations/011_local_sensitivity_analysis.ts

@@ -1,42 +0,0 @@
-import { sql, type Kysely } from "kysely";
-import type { CatalogDatabase } from "../repository.js";
-
-export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
-  await sql.raw(`alter table sensitive_data_suggestion_runs
-    alter column model_id drop not null,
-    add column engine text not null default 'llm',
-    add column policy_version text,
-    add column unknown integer not null default 0,
-    drop constraint sensitive_data_suggestion_runs_counters_check,
-    add constraint sensitive_data_suggestion_runs_counters_check
-      check (total >= 0
-        and suggested_sensitive >= 0
-        and suggested_non_sensitive >= 0
-        and unknown >= 0
-        and suggested_sensitive + suggested_non_sensitive + unknown <= total),
-    add constraint sensitive_data_suggestion_runs_engine_check
-      check (engine in ('llm', 'local')),
-    add constraint sensitive_data_suggestion_runs_origin_check
-      check ((engine = 'llm' and model_id is not null and policy_version is null)
-        or (engine = 'local' and model_id is null
-          and policy_version ~ '^[a-z][a-z0-9._-]{0,63}$'))`).execute(db);
-}
-
-export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
-  await sql.raw(`alter table sensitive_data_suggestion_runs
-    drop constraint sensitive_data_suggestion_runs_origin_check,
-    drop constraint sensitive_data_suggestion_runs_engine_check,
-    drop constraint sensitive_data_suggestion_runs_counters_check`).execute(db);
-  await sql.raw(`update sensitive_data_suggestion_runs
-    set model_id = coalesce(model_id, 'local/sensitivity-v1')`).execute(db);
-  await sql.raw(`alter table sensitive_data_suggestion_runs
-    drop column unknown,
-    drop column policy_version,
-    drop column engine,
-    alter column model_id set not null,
-    add constraint sensitive_data_suggestion_runs_counters_check
-      check (total >= 0
-        and suggested_sensitive >= 0
-        and suggested_non_sensitive >= 0
-        and suggested_sensitive + suggested_non_sensitive <= total)`).execute(db);
-}

backend/src/catalog/migrations/012_sensitivity_reason.ts

@@ -1,12 +0,0 @@
-import type { Kysely } from "kysely";
-import type { CatalogDatabase } from "../repository.js";
-
-export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
-  await db.schema.alterTable("catalog_columns")
-    .addColumn("sensitivity_reason", "text")
-    .execute();
-}
-
-export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
-  await db.schema.alterTable("catalog_columns").dropColumn("sensitivity_reason").execute();
-}

backend/src/catalog/migrations/013_catalog_preprocessing_state.ts

@@ -1,212 +0,0 @@
-import { type Kysely, sql } from "kysely";
-import type { CatalogDatabase } from "../repository.js";
-
-export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
-  await db.schema.alterTable("workspace_databases")
-    .addColumn("metadata_content_revision", "bigint", (column) => column.notNull().defaultTo(0))
-    .addColumn("preprocessing_status", "text", (column) => column.notNull().defaultTo("failed"))
-    .addColumn("preprocessing_input_fingerprint", "text")
-    .addColumn("preprocessed_metadata_revision", "bigint")
-    .addColumn("preprocessing_started_at", "timestamptz")
-    .addColumn("preprocessing_finished_at", "timestamptz")
-    .addColumn("preprocessing_error_code", "text")
-    .execute();
-  await sql`
-    alter table workspace_databases
-    add constraint workspace_databases_preprocessing_status_check
-    check (preprocessing_status in ('running', 'succeeded', 'failed'))
-  `.execute(db);
-
-  await sql`
-    create function catalog_metadata_write_guard()
-    returns trigger
-    language plpgsql
-    as $$
-    declare
-      resolved_database_id uuid;
-      current_status text;
-      relation_id uuid;
-      table_id uuid;
-    begin
-      if tg_table_name = 'catalog_tables' then
-        resolved_database_id := coalesce(new.database_id, old.database_id);
-      elsif tg_table_name = 'catalog_columns' then
-        table_id := coalesce(new.table_id, old.table_id);
-        select database_id into resolved_database_id from catalog_tables where id = table_id;
-      elsif tg_table_name = 'catalog_relationships' then
-        resolved_database_id := coalesce(new.database_id, old.database_id);
-      elsif tg_table_name = 'catalog_relationship_columns' then
-        relation_id := coalesce(new.relationship_id, old.relationship_id);
-        select database_id into resolved_database_id from catalog_relationships where id = relation_id;
-      elsif tg_table_name = 'catalog_logical_relationships' then
-        resolved_database_id := coalesce(new.database_id, old.database_id);
-      elsif tg_table_name = 'database_bindings' then
-        if tg_op = 'UPDATE' and not (
-          new.transport is distinct from old.transport
-          or new.host is distinct from old.host
-          or new.port is distinct from old.port
-          or new.username is distinct from old.username
-          or new.base_url is distinct from old.base_url
-          or new.rest_path is distinct from old.rest_path
-          or new.rest_auth is distinct from old.rest_auth
-          or new.tls_servername is distinct from old.tls_servername
-          or new.ssh_host is distinct from old.ssh_host
-          or new.ssh_port is distinct from old.ssh_port
-          or new.ssh_username is distinct from old.ssh_username
-          or new.ssh_target_host is distinct from old.ssh_target_host
-          or new.ssh_target_port is distinct from old.ssh_target_port
-        ) then
-          return new;
-        end if;
-        resolved_database_id := coalesce(new.database_id, old.database_id);
-      end if;
-      if resolved_database_id is null then
-        if tg_op = 'DELETE' then return old; else return new; end if;
-      end if;
-
-      select preprocessing_status into current_status
-      from workspace_databases
-      where id = resolved_database_id
-      for update;
-
-      if current_status = 'running' then
-        raise exception 'catalog preprocessing is running'
-          using errcode = '55000';
-      end if;
-
-      update workspace_databases
-      set metadata_content_revision = metadata_content_revision + 1,
-          preprocessing_status = 'failed',
-          preprocessing_finished_at = now(),
-          preprocessing_error_code = 'catalog_changed',
-          updated_at = now()
-      where id = resolved_database_id;
-      if tg_op = 'DELETE' then return old; else return new; end if;
-    end;
-    $$
-  `.execute(db);
-
-  for (const table of [
-    "catalog_tables",
-    "catalog_columns",
-    "catalog_relationships",
-    "catalog_relationship_columns",
-    "catalog_logical_relationships",
-    "database_bindings",
-  ]) {
-    await sql.raw(`
-      create trigger ${table}_metadata_write_guard
-      before insert or update or delete on ${table}
-      for each row execute function catalog_metadata_write_guard()
-    `).execute(db);
-  }
-
-  await sql`
-    create function catalog_database_configuration_guard()
-    returns trigger
-    language plpgsql
-    as $$
-    begin
-      if new.workspace_id is distinct from old.workspace_id
-        or new.engine is distinct from old.engine
-        or new.database_name is distinct from old.database_name
-        or new.schema_name is distinct from old.schema_name then
-        if old.preprocessing_status = 'running' then
-          raise exception 'catalog preprocessing is running'
-            using errcode = '55000';
-        end if;
-        new.metadata_content_revision := old.metadata_content_revision + 1;
-        new.preprocessing_status := 'failed';
-        new.preprocessing_finished_at := now();
-        new.preprocessing_error_code := 'catalog_changed';
-      end if;
-      return new;
-    end;
-    $$
-  `.execute(db);
-  await sql`
-    create trigger workspace_databases_configuration_guard
-    before update of workspace_id, engine, database_name, schema_name on workspace_databases
-    for each row execute function catalog_database_configuration_guard()
-  `.execute(db);
-
-  await sql`
-    create function catalog_operation_start_guard()
-    returns trigger
-    language plpgsql
-    as $$
-    declare
-      current_status text;
-      starting boolean;
-    begin
-      if tg_table_name = 'catalog_sync_runs' then
-        starting := new.state in ('queued', 'running', 'awaiting_confirmation', 'applying');
-      else
-        starting := new.status in ('queued', 'running');
-      end if;
-      if not starting then
-        return new;
-      end if;
-
-      select preprocessing_status into current_status
-      from workspace_databases
-      where id = new.database_id
-      for update;
-      if current_status = 'running' then
-        raise exception 'catalog preprocessing is running'
-          using errcode = '55000';
-      end if;
-      return new;
-    end;
-    $$
-  `.execute(db);
-
-  for (const table of [
-    "catalog_sync_runs",
-    "description_generation_runs",
-    "sensitive_data_suggestion_runs",
-  ]) {
-    await sql.raw(`
-      create trigger ${table}_operation_start_guard
-      before insert or update on ${table}
-      for each row execute function catalog_operation_start_guard()
-    `).execute(db);
-  }
-}
-
-export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
-  for (const table of [
-    "catalog_sync_runs",
-    "description_generation_runs",
-    "sensitive_data_suggestion_runs",
-  ]) {
-    await sql.raw(`drop trigger if exists ${table}_operation_start_guard on ${table}`).execute(db);
-  }
-  await sql`drop function if exists catalog_operation_start_guard()`.execute(db);
-  await sql`drop trigger if exists workspace_databases_configuration_guard on workspace_databases`.execute(db);
-  await sql`drop function if exists catalog_database_configuration_guard()`.execute(db);
-  for (const table of [
-    "catalog_tables",
-    "catalog_columns",
-    "catalog_relationships",
-    "catalog_relationship_columns",
-    "catalog_logical_relationships",
-    "database_bindings",
-  ]) {
-    await sql.raw(`drop trigger if exists ${table}_metadata_write_guard on ${table}`).execute(db);
-  }
-  await sql`drop function if exists catalog_metadata_write_guard()`.execute(db);
-  await db.schema.alterTable("workspace_databases")
-    .dropConstraint("workspace_databases_preprocessing_status_check").execute();
-  for (const column of [
-    "preprocessing_error_code",
-    "preprocessing_finished_at",
-    "preprocessing_started_at",
-    "preprocessed_metadata_revision",
-    "preprocessing_input_fingerprint",
-    "preprocessing_status",
-    "metadata_content_revision",
-  ]) {
-    await sql.raw(`alter table workspace_databases drop column ${column}`).execute(db);
-  }
-}

backend/src/catalog/repository.ts

@@ -27,8 +27,6 @@ import {
   type CatalogLogicalRelationshipCandidate,
   type CatalogLogicalRelationshipContext,
   type CatalogPhysicalRelationship,
-  type CatalogPreprocessingStartResult,
-  type CatalogPreprocessingClearResult,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -47,20 +45,15 @@ import {
   type DescriptionGenerationScope,
   type ObservedCatalogTable,
   type ObservedSchemaSnapshot,
-  type SensitivityAnalysisEvent,
-  type SensitivityAnalysisRun,
-  type SensitivityAnalysisRunUpdate,
-  type SensitivityAnalysisScope,
+  type SensitiveDataSuggestionEvent,
+  type SensitiveDataSuggestionRun,
+  type SensitiveDataSuggestionRunUpdate,
+  type SensitiveDataSuggestionScope,
   type TableSyncRepositoryResult,
   type WorkspaceDatabase,
 } from "./types.js";
 
 type Timestamp = ColumnType<Date, Date | string | undefined, Date | string>;
-type NullableTimestamp = ColumnType<
-  Date | null,
-  Date | string | null | undefined,
-  Date | string | null
->;
 
 interface WorkspaceDatabaseTable {
   id: string;
@@ -73,13 +66,6 @@ interface WorkspaceDatabaseTable {
   updatedAt: Timestamp;
   schemaSyncedVersion: number | null;
   schemaSyncedAt: Timestamp | null;
-  metadataContentRevision: Generated<number>;
-  preprocessingStatus: Generated<WorkspaceDatabase["preprocessingStatus"]>;
-  preprocessingInputFingerprint: Generated<string | null>;
-  preprocessedMetadataRevision: Generated<number | null>;
-  preprocessingStartedAt: NullableTimestamp;
-  preprocessingFinishedAt: NullableTimestamp;
-  preprocessingErrorCode: Generated<string | null>;
 }
 
 interface DatabaseBindingTable {
@@ -132,7 +118,6 @@ interface CatalogColumnTable {
   description: string | null;
   generatedDescription: string | null;
   sensitive: Generated<boolean>;
-  sensitivityReason: Generated<string | null>;
   lastSyncedDatabaseVersion: number | null;
   lastSyncedAt: Timestamp | null;
   version: Generated<number>;
@@ -204,18 +189,15 @@ interface DescriptionGenerationEventTable {
   createdAt: Timestamp;
 }
 
-interface SensitivityAnalysisRunTable {
+interface SensitiveDataSuggestionRunTable {
   id: string;
   databaseId: string;
-  scope: SensitivityAnalysisScope;
-  engine: SensitivityAnalysisRun["engine"];
-  modelId: string | null;
-  policyVersion: string | null;
-  status: SensitivityAnalysisRun["status"];
+  scope: SensitiveDataSuggestionScope;
+  modelId: string;
+  status: SensitiveDataSuggestionRun["status"];
   total: number;
   suggestedSensitive: number;
   suggestedNonSensitive: number;
-  unknown: number;
   inputTokens: number;
   cacheReadTokens: number;
   outputTokens: number;
@@ -226,10 +208,10 @@ interface SensitivityAnalysisRunTable {
   errorSummary: string | null;
 }
 
-interface SensitivityAnalysisEventTable {
+interface SensitiveDataSuggestionEventTable {
   runId: string;
   sequence: number;
-  level: SensitivityAnalysisEvent["level"];
+  level: SensitiveDataSuggestionEvent["level"];
   message: string;
   createdAt: Timestamp;
 }
@@ -282,9 +264,8 @@ export interface CatalogDatabase {
   catalogLogicalRelationships: CatalogLogicalRelationshipTable;
   descriptionGenerationRuns: DescriptionGenerationRunTable;
   descriptionGenerationEvents: DescriptionGenerationEventTable;
-  // Legacy physical table names retained for migration and storage compatibility.
-  sensitiveDataSuggestionRuns: SensitivityAnalysisRunTable;
-  sensitiveDataSuggestionEvents: SensitivityAnalysisEventTable;
+  sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable;
+  sensitiveDataSuggestionEvents: SensitiveDataSuggestionEventTable;
   catalogSyncRuns: CatalogSyncRunTable;
   catalogSyncEvents: CatalogSyncEventTable;
 }
@@ -339,19 +320,6 @@ function serialize(row: JoinedRow): WorkspaceDatabase {
     lastErrorMessage: present(row.lastErrorMessage),
     schemaSyncedVersion: present(row.schemaSyncedVersion),
     schemaSyncedAt: row.schemaSyncedAt == null ? undefined : new Date(row.schemaSyncedAt).toISOString(),
-    metadataContentRevision: Number(row.metadataContentRevision),
-    preprocessingStatus: row.preprocessingStatus,
-    preprocessingInputFingerprint: present(row.preprocessingInputFingerprint),
-    preprocessedMetadataRevision: row.preprocessedMetadataRevision == null
-      ? undefined
-      : Number(row.preprocessedMetadataRevision),
-    preprocessingStartedAt: row.preprocessingStartedAt == null
-      ? undefined
-      : new Date(row.preprocessingStartedAt).toISOString(),
-    preprocessingFinishedAt: row.preprocessingFinishedAt == null
-      ? undefined
-      : new Date(row.preprocessingFinishedAt).toISOString(),
-    preprocessingErrorCode: present(row.preprocessingErrorCode),
   };
 }
 
@@ -388,7 +356,6 @@ function serializeColumn(row: Selectable<CatalogColumnTable>, foreignKeyCount =
     description: row.description,
     generatedDescription: row.generatedDescription,
     sensitive: row.sensitive,
-    sensitivityReason: row.sensitivityReason,
     lastSyncedDatabaseVersion: row.lastSyncedDatabaseVersion,
     lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(),
     version: row.version,
@@ -435,9 +402,9 @@ function serializeDescriptionGenerationEvent(
   return { ...row, createdAt: new Date(row.createdAt).toISOString() };
 }
 
-function serializeSensitivityAnalysisRun(
-  row: Selectable<SensitivityAnalysisRunTable>,
-): SensitivityAnalysisRun {
+function serializeSensitiveDataSuggestionRun(
+  row: Selectable<SensitiveDataSuggestionRunTable>,
+): SensitiveDataSuggestionRun {
   const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString();
   return {
     ...row,
@@ -448,9 +415,9 @@ function serializeSensitivityAnalysisRun(
   };
 }
 
-function serializeSensitivityAnalysisEvent(
-  row: Selectable<SensitivityAnalysisEventTable>,
-): SensitivityAnalysisEvent {
+function serializeSensitiveDataSuggestionEvent(
+  row: Selectable<SensitiveDataSuggestionEventTable>,
+): SensitiveDataSuggestionEvent {
   return { ...row, createdAt: new Date(row.createdAt).toISOString() };
 }
 
@@ -504,98 +471,6 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return id ? await this.get(id.id) : undefined;
   }
 
-  async beginPreprocessing(
-    workspaceId: string,
-    inputFingerprint: string,
-  ): Promise<CatalogPreprocessingStartResult> {
-    return await this.db.transaction().execute(async (trx) => {
-      const database = await trx.selectFrom("workspaceDatabases")
-        .selectAll()
-        .where("workspaceId", "=", workspaceId)
-        .forUpdate()
-        .executeTakeFirst();
-      if (!database) return { kind: "not_found" };
-      if (database.preprocessingStatus === "running") return { kind: "already_running" };
-      if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
-
-      const activeSync = await trx.selectFrom("catalogSyncRuns")
-        .select("id")
-        .where("databaseId", "=", database.id)
-        .where("state", "in", ["queued", "running", "awaiting_confirmation", "applying"])
-        .executeTakeFirst();
-      const activeDescriptions = await trx.selectFrom("descriptionGenerationRuns")
-        .select("id")
-        .where("databaseId", "=", database.id)
-        .where("status", "in", ["queued", "running"])
-        .executeTakeFirst();
-      const activeSensitivity = await trx.selectFrom("sensitiveDataSuggestionRuns")
-        .select("id")
-        .where("databaseId", "=", database.id)
-        .where("status", "=", "running")
-        .executeTakeFirst();
-      if (activeSync || activeDescriptions || activeSensitivity) return { kind: "catalog_busy" };
-
-      await trx.updateTable("workspaceDatabases")
-        .set({
-          preprocessingStatus: "running",
-          preprocessingInputFingerprint: inputFingerprint,
-          preprocessedMetadataRevision: null,
-          preprocessingStartedAt: sql`now()`,
-          preprocessingFinishedAt: null,
-          preprocessingErrorCode: null,
-          updatedAt: sql`now()`,
-        })
-        .where("id", "=", database.id)
-        .execute();
-      return { kind: "started", database: (await selectOne(trx, database.id))! };
-    });
-  }
-
-  async finishPreprocessing(
-    workspaceId: string,
-    metadataContentRevision: number,
-    inputFingerprint: string,
-    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
-  ): Promise<WorkspaceDatabase | undefined> {
-    const result = await this.db.updateTable("workspaceDatabases")
-      .set({
-        preprocessingStatus: outcome.status,
-        preprocessedMetadataRevision: outcome.status === "succeeded" ? metadataContentRevision : null,
-        preprocessingFinishedAt: sql`now()`,
-        preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : null,
-        updatedAt: sql`now()`,
-      })
-      .where("workspaceId", "=", workspaceId)
-      .where("preprocessingStatus", "=", "running")
-      .where("metadataContentRevision", "=", metadataContentRevision)
-      .where("preprocessingInputFingerprint", "=", inputFingerprint)
-      .returning("id")
-      .executeTakeFirst();
-    return result ? await this.get(result.id) : undefined;
-  }
-
-  async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
-    return await this.db.transaction().execute(async (trx) => {
-      const database = await trx.selectFrom("workspaceDatabases")
-        .select(["id", "preprocessingStatus"])
-        .where("workspaceId", "=", workspaceId)
-        .forUpdate()
-        .executeTakeFirst();
-      if (!database) return { kind: "not_found" };
-      if (database.preprocessingStatus === "running") return { kind: "already_running" };
-      await trx.updateTable("workspaceDatabases").set({
-        preprocessingStatus: "failed",
-        preprocessingInputFingerprint: null,
-        preprocessedMetadataRevision: null,
-        preprocessingStartedAt: null,
-        preprocessingFinishedAt: sql`now()`,
-        preprocessingErrorCode: "derived_data_cleared",
-        updatedAt: sql`now()`,
-      }).where("id", "=", database.id).execute();
-      return { kind: "cleared", database: (await selectOne(trx, database.id))! };
-    });
-  }
-
   async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
     const result = await sql<CatalogMetricsRow>`
       WITH requested_database AS (
@@ -637,18 +512,10 @@ export class KyselyCatalogRepository implements CatalogRepository {
       relationship_metrics AS (
         SELECT
           count(*)::int AS relationships,
-          max(relationship.updated_at) AS updated_at
-        FROM (
-          SELECT catalog_relationships.updated_at
-          FROM catalog_relationships
-          INNER JOIN selected_databases
-            ON selected_databases.id = catalog_relationships.database_id
-          UNION ALL
-          SELECT catalog_logical_relationships.updated_at
-          FROM catalog_logical_relationships
-          INNER JOIN selected_databases
-            ON selected_databases.id = catalog_logical_relationships.database_id
-        ) AS relationship
+          max(catalog_relationships.updated_at) AS updated_at
+        FROM catalog_relationships
+        INNER JOIN selected_databases
+          ON selected_databases.id = catalog_relationships.database_id
       )
       SELECT
         (SELECT count(*)::int FROM selected_databases) AS "databaseCount",
@@ -857,7 +724,6 @@ export class KyselyCatalogRepository implements CatalogRepository {
     description: string | null,
     generatedDescription: string | null,
     sensitive?: boolean,
-    sensitivityReason?: string | null,
   ): Promise<CatalogColumn | undefined> {
     const belongs = await this.db.selectFrom("catalogTables").select("id")
       .where("id", "=", tableId).where("databaseId", "=", databaseId).executeTakeFirst();
@@ -866,9 +732,6 @@ export class KyselyCatalogRepository implements CatalogRepository {
       description,
       generatedDescription,
       ...(sensitive === undefined ? {} : { sensitive }),
-      ...(sensitive === false
-        ? { sensitivityReason: null }
-        : sensitivityReason === undefined ? {} : { sensitivityReason }),
       version: sql`version + 1`,
       updatedAt: sql`now()`,
     }).where("id", "=", columnId).where("tableId", "=", tableId)
@@ -882,9 +745,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
     targetIds: readonly string[],
   ): Promise<CatalogDescriptionConsolidationCounts | undefined> {
     const selectedTargetIds = [...new Set(targetIds)];
-    if (target !== "database" && target !== "database_columns" && selectedTargetIds.length === 0) {
-      return undefined;
-    }
+    if (selectedTargetIds.length === 0) return undefined;
     return await this.db.transaction().execute(async (trx) => {
       const database = await trx.selectFrom("workspaceDatabases").select("id")
         .where("id", "=", databaseId).forUpdate().executeTakeFirst();
@@ -914,51 +775,29 @@ export class KyselyCatalogRepository implements CatalogRepository {
         };
       }
 
-      const tableRows = await trx.selectFrom("catalogTables").select(["id", "generatedDescription"])
-        .where("databaseId", "=", databaseId)
-        .orderBy("id")
-        .forUpdate()
-        .execute();
-      const copiedTableIds = target === "database"
-        ? tableRows
-          .filter((row) => Boolean(row.generatedDescription?.trim()))
-          .map((row) => row.id)
-        : [];
-      if (copiedTableIds.length > 0) {
-        await trx.updateTable("catalogTables").set({
-          description: sql`generated_description`,
-          version: sql`version + 1`,
-          updatedAt: sql`now()`,
-        }).where("id", "in", copiedTableIds).execute();
-      }
+      const tableRows = await trx.selectFrom("catalogTables").select("id")
+        .where("databaseId", "=", databaseId).execute();
       const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
         .select(["id", "generatedDescription"])
         .where("tableId", "in", tableRows.map((table) => table.id))
-        .$if(target === "columns", (query) => query.where("id", "in", selectedTargetIds))
+        .where("id", "in", selectedTargetIds)
         .orderBy("id")
         .forUpdate()
         .execute();
-      if (target === "columns" && rows.length !== selectedTargetIds.length) return undefined;
+      if (rows.length !== selectedTargetIds.length) return undefined;
       const copiedIds = rows
         .filter((row) => Boolean(row.generatedDescription?.trim()))
         .map((row) => row.id);
       if (copiedIds.length > 0) {
-        let update = trx.updateTable("catalogColumns").set({
+        await trx.updateTable("catalogColumns").set({
           description: sql`generated_description`,
           version: sql`version + 1`,
           updatedAt: sql`now()`,
-        });
-        update = target === "database" || target === "database_columns"
-          ? update
-            .where("tableId", "in", tableRows.map((table) => table.id))
-            .where(sql<boolean>`nullif(btrim(generated_description), '') is not null`)
-          : update.where("id", "in", copiedIds);
-        await update.execute();
+        }).where("id", "in", copiedIds).execute();
       }
       return {
-        copied: copiedTableIds.length + copiedIds.length,
-        skipped: (target === "database" ? tableRows.length : 0) - copiedTableIds.length
-          + rows.length - copiedIds.length,
+        copied: copiedIds.length,
+        skipped: selectedTargetIds.length - copiedIds.length,
       };
     });
   }
@@ -1099,55 +938,52 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return rows.map(serializeDescriptionGenerationEvent);
   }
 
-  async createSensitivityAnalysisRun(
+  async createSensitiveDataSuggestionRun(
     databaseId: string,
-    scope: SensitivityAnalysisScope,
-    origin: { engine: "llm"; modelId: string } | { engine: "local"; policyVersion: string },
-  ): Promise<SensitivityAnalysisRun> {
+    scope: SensitiveDataSuggestionScope,
+    modelId: string,
+  ): Promise<SensitiveDataSuggestionRun> {
     const row = await this.db.insertInto("sensitiveDataSuggestionRuns").values({
       id: randomUUID(),
       databaseId,
       scope,
-      engine: origin.engine,
-      modelId: origin.engine === "llm" ? origin.modelId : null,
-      policyVersion: origin.engine === "local" ? origin.policyVersion : null,
+      modelId,
       status: "running",
       total: 0,
       suggestedSensitive: 0,
       suggestedNonSensitive: 0,
-      unknown: 0,
       inputTokens: 0,
       cacheReadTokens: 0,
       outputTokens: 0,
       finishedAt: null,
       errorSummary: null,
     }).returningAll().executeTakeFirstOrThrow();
-    return serializeSensitivityAnalysisRun(row);
+    return serializeSensitiveDataSuggestionRun(row);
   }
 
-  async getSensitivityAnalysisRun(
+  async getSensitiveDataSuggestionRun(
     runId: string,
-  ): Promise<SensitivityAnalysisRun | undefined> {
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
     const row = await this.db.selectFrom("sensitiveDataSuggestionRuns")
       .selectAll()
       .where("id", "=", runId)
       .executeTakeFirst();
-    return row ? serializeSensitivityAnalysisRun(row) : undefined;
+    return row ? serializeSensitiveDataSuggestionRun(row) : undefined;
   }
 
-  async listSensitivityAnalysisRuns(limit = 50): Promise<SensitivityAnalysisRun[]> {
+  async listSensitiveDataSuggestionRuns(limit = 50): Promise<SensitiveDataSuggestionRun[]> {
     const rows = await this.db.selectFrom("sensitiveDataSuggestionRuns")
       .selectAll()
       .orderBy("createdAt", "desc")
       .orderBy("id", "desc")
       .limit(limit)
       .execute();
-    return rows.map(serializeSensitivityAnalysisRun);
+    return rows.map(serializeSensitiveDataSuggestionRun);
   }
 
-  async interruptActiveSensitivityAnalysisRuns(
+  async interruptActiveSensitiveDataSuggestionRuns(
     errorSummary: string,
-  ): Promise<SensitivityAnalysisRun[]> {
+  ): Promise<SensitiveDataSuggestionRun[]> {
     const rows = await this.db.updateTable("sensitiveDataSuggestionRuns")
       .set({
         status: "interrupted",
@@ -1158,34 +994,34 @@ export class KyselyCatalogRepository implements CatalogRepository {
       .where("status", "=", "running")
       .returningAll()
       .execute();
-    return rows.map(serializeSensitivityAnalysisRun);
+    return rows.map(serializeSensitiveDataSuggestionRun);
   }
 
-  async updateSensitivityAnalysisRun(
+  async updateSensitiveDataSuggestionRun(
     runId: string,
-    update: SensitivityAnalysisRunUpdate,
-  ): Promise<SensitivityAnalysisRun | undefined> {
+    update: SensitiveDataSuggestionRunUpdate,
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
     const values: any = { ...update, updatedAt: sql`now()` };
     const row = await this.db.updateTable("sensitiveDataSuggestionRuns")
       .set(values)
       .where("id", "=", runId)
       .returningAll()
       .executeTakeFirst();
-    return row ? serializeSensitivityAnalysisRun(row) : undefined;
+    return row ? serializeSensitiveDataSuggestionRun(row) : undefined;
   }
 
-  async appendSensitivityAnalysisEvent(
+  async appendSensitiveDataSuggestionEvent(
     runId: string,
-    level: SensitivityAnalysisEvent["level"],
+    level: SensitiveDataSuggestionEvent["level"],
     message: string,
-  ): Promise<SensitivityAnalysisEvent> {
+  ): Promise<SensitiveDataSuggestionEvent> {
     return await this.db.transaction().execute(async (trx) => {
       const run = await trx.selectFrom("sensitiveDataSuggestionRuns")
         .select("id")
         .where("id", "=", runId)
         .forUpdate()
         .executeTakeFirst();
-      if (!run) throw new CatalogConflictError("Sensitivity Analysis Run does not exist");
+      if (!run) throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist");
       const current = await trx.selectFrom("sensitiveDataSuggestionEvents")
         .select(sql<number>`coalesce(max(sequence), 0)::int`.as("sequence"))
         .where("runId", "=", runId)
@@ -1196,21 +1032,21 @@ export class KyselyCatalogRepository implements CatalogRepository {
         level,
         message,
       }).returningAll().executeTakeFirstOrThrow();
-      return serializeSensitivityAnalysisEvent(row);
+      return serializeSensitiveDataSuggestionEvent(row);
     });
   }
 
-  async listSensitivityAnalysisEvents(
+  async listSensitiveDataSuggestionEvents(
     runId: string,
     afterSequence = 0,
-  ): Promise<SensitivityAnalysisEvent[]> {
+  ): Promise<SensitiveDataSuggestionEvent[]> {
     const rows = await this.db.selectFrom("sensitiveDataSuggestionEvents")
       .selectAll()
       .where("runId", "=", runId)
       .where("sequence", ">", afterSequence)
       .orderBy("sequence")
       .execute();
-    return rows.map(serializeSensitivityAnalysisEvent);
+    return rows.map(serializeSensitiveDataSuggestionEvent);
   }
 
   async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
@@ -1418,25 +1254,16 @@ export class KyselyCatalogRepository implements CatalogRepository {
       if (databases.length !== selectedDatabaseIds.length) return undefined;
 
       if (target === "relationships") {
-        const physicalCount = await trx.selectFrom("catalogRelationships")
+        const count = await trx.selectFrom("catalogRelationships")
           .select(sql<number>`count(*)::int`.as("count"))
           .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
-        const logicalCount = await trx.selectFrom("catalogLogicalRelationships")
-          .select(sql<number>`count(*)::int`.as("count"))
-          .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
-        await trx.deleteFrom("catalogLogicalRelationships")
-          .where("databaseId", "in", selectedDatabaseIds).execute();
         await trx.deleteFrom("catalogRelationships")
           .where("databaseId", "in", selectedDatabaseIds).execute();
         await trx.updateTable("workspaceDatabases").set({
           schemaSyncedVersion: null,
           schemaSyncedAt: null,
         }).where("id", "in", selectedDatabaseIds).execute();
-        return {
-          tables: 0,
-          columns: 0,
-          relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0),
-        };
+        return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
       }
 
       const tableCount = await trx.selectFrom("catalogTables")
@@ -1446,10 +1273,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
         .innerJoin("catalogTables", "catalogTables.id", "catalogColumns.tableId")
         .select(sql<number>`count(*)::int`.as("count"))
         .where("catalogTables.databaseId", "in", selectedDatabaseIds).executeTakeFirst();
-      const physicalRelationshipCount = await trx.selectFrom("catalogRelationships")
-        .select(sql<number>`count(*)::int`.as("count"))
-        .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
-      const logicalRelationshipCount = await trx.selectFrom("catalogLogicalRelationships")
+      const relationshipCount = await trx.selectFrom("catalogRelationships")
         .select(sql<number>`count(*)::int`.as("count"))
         .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
       await trx.deleteFrom("catalogTables")
@@ -1461,8 +1285,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
       return {
         tables: Number(tableCount?.count ?? 0),
         columns: Number(columnCount?.count ?? 0),
-        relationships: Number(physicalRelationshipCount?.count ?? 0)
-          + Number(logicalRelationshipCount?.count ?? 0),
+        relationships: Number(relationshipCount?.count ?? 0),
       };
     });
   }
@@ -1496,34 +1319,13 @@ export class KyselyCatalogRepository implements CatalogRepository {
         return { tables: 0, columns: Number(count?.count ?? 0), relationships: 0 };
       }
 
-      const physicalCount = await trx.selectFrom("catalogRelationships")
+      const count = await trx.selectFrom("catalogRelationships")
         .select(sql<number>`count(*)::int`.as("count"))
         .where("databaseId", "=", databaseId)
         .where((eb) => eb.or([
           eb("sourceTableId", "in", selectedTableIds),
           eb("targetTableId", "in", selectedTableIds),
         ])).executeTakeFirst();
-      const selectedColumnIds = (await trx.selectFrom("catalogColumns")
-        .select("id")
-        .where("tableId", "in", selectedTableIds)
-        .execute()).map((column) => column.id);
-      const logicalCount = selectedColumnIds.length === 0
-        ? undefined
-        : await trx.selectFrom("catalogLogicalRelationships")
-            .select(sql<number>`count(*)::int`.as("count"))
-            .where("databaseId", "=", databaseId)
-            .where((eb) => eb.or([
-              eb("sourceColumnId", "in", selectedColumnIds),
-              eb("targetColumnId", "in", selectedColumnIds),
-            ])).executeTakeFirst();
-      if (selectedColumnIds.length > 0) {
-        await trx.deleteFrom("catalogLogicalRelationships")
-          .where("databaseId", "=", databaseId)
-          .where((eb) => eb.or([
-            eb("sourceColumnId", "in", selectedColumnIds),
-            eb("targetColumnId", "in", selectedColumnIds),
-          ])).execute();
-      }
       await trx.deleteFrom("catalogRelationships")
         .where("databaseId", "=", databaseId)
         .where((eb) => eb.or([
@@ -1534,11 +1336,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
         schemaSyncedVersion: null,
         schemaSyncedAt: null,
       }).where("id", "=", databaseId).execute();
-      return {
-        tables: 0,
-        columns: 0,
-        relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0),
-      };
+      return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
     });
   }
 
@@ -1972,9 +1770,6 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async list(): Promise<WorkspaceDatabase[]> { return this.fail(); }
   async get(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
   async getByWorkspace(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
-  async beginPreprocessing(): Promise<CatalogPreprocessingStartResult> { return this.fail(); }
-  async finishPreprocessing(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
-  async clearPreprocessing(): Promise<CatalogPreprocessingClearResult> { return this.fail(); }
   async getCatalogMetrics(): Promise<CatalogMetrics | undefined> { return this.fail(); }
   async create(): Promise<WorkspaceDatabase> { return this.fail(); }
   async update(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
@@ -1997,13 +1792,13 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async updateDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
   async appendDescriptionGenerationEvent(): Promise<DescriptionGenerationEvent> { return this.fail(); }
   async listDescriptionGenerationEvents(): Promise<DescriptionGenerationEvent[]> { return this.fail(); }
-  async createSensitivityAnalysisRun(): Promise<SensitivityAnalysisRun> { return this.fail(); }
-  async getSensitivityAnalysisRun(): Promise<SensitivityAnalysisRun | undefined> { return this.fail(); }
-  async listSensitivityAnalysisRuns(): Promise<SensitivityAnalysisRun[]> { return this.fail(); }
-  async interruptActiveSensitivityAnalysisRuns(): Promise<SensitivityAnalysisRun[]> { return this.fail(); }
-  async updateSensitivityAnalysisRun(): Promise<SensitivityAnalysisRun | undefined> { return this.fail(); }
-  async appendSensitivityAnalysisEvent(): Promise<SensitivityAnalysisEvent> { return this.fail(); }
-  async listSensitivityAnalysisEvents(): Promise<SensitivityAnalysisEvent[]> { return this.fail(); }
+  async createSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun> { return this.fail(); }
+  async getSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
+  async listSensitiveDataSuggestionRuns(): Promise<SensitiveDataSuggestionRun[]> { return this.fail(); }
+  async interruptActiveSensitiveDataSuggestionRuns(): Promise<SensitiveDataSuggestionRun[]> { return this.fail(); }
+  async updateSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
+  async appendSensitiveDataSuggestionEvent(): Promise<SensitiveDataSuggestionEvent> { return this.fail(); }
+  async listSensitiveDataSuggestionEvents(): Promise<SensitiveDataSuggestionEvent[]> { return this.fail(); }
   async listRelationships(): Promise<CatalogPhysicalRelationship[]> { return this.fail(); }
   async listLogicalRelationships(): Promise<CatalogLogicalRelationship[]> { return this.fail(); }
   async getLogicalRelationshipContext(): Promise<CatalogLogicalRelationshipContext | undefined> { return this.fail(); }

backend/src/catalog/runtime-binding.ts

@@ -1,140 +0,0 @@
-import { dirname } from "node:path";
-
-import { resolveRuntimeBindings, type RuntimeBindings } from "../workspaces/bindings.js";
-import { buildInstallationContract, type InstallationSuffix } from "../workspaces/contracts.js";
-import {
-  discoverWorkspaceSecretRequirements,
-} from "../workspaces/secret-requirements.js";
-import type {
-  WorkspaceSecretMaterialization,
-  WorkspaceSecretStore,
-} from "../workspaces/secret-store.js";
-import {
-  validateWorkspaceDescriptor,
-  type WorkspaceDescriptor,
-} from "../workspaces/schema.js";
-import { CATALOG_SECRET_IDS } from "./secrets.js";
-import type { WorkspaceDatabase } from "./types.js";
-
-export interface CatalogRuntimeBindingLease {
-  workspace: WorkspaceDescriptor;
-  bindings: RuntimeBindings;
-  release(): void;
-}
-
-const CATALOG_SECRET_BY_SUFFIX: Readonly<Partial<Record<InstallationSuffix, string>>> = {
-  PASSWORD_FILE: CATALOG_SECRET_IDS.password,
-  API_KEY_FILE: CATALOG_SECRET_IDS.apiKey,
-  TLS_CA_FILE: CATALOG_SECRET_IDS.tlsCa,
-  SSH_PRIVATE_KEY_FILE: CATALOG_SECRET_IDS.sshPrivateKey,
-  SSH_KNOWN_HOSTS_FILE: CATALOG_SECRET_IDS.sshKnownHosts,
-};
-
-function runtimeWorkspace(
-  workspace: WorkspaceDescriptor,
-  database: WorkspaceDatabase,
-): WorkspaceDescriptor {
-  if (workspace.workspace.id !== database.workspaceId) {
-    throw new Error("Catalog database binding does not belong to the workspace");
-  }
-  const { dwh: _legacyDwh, diagnostics: _legacyDiagnostics, ...descriptor } = workspace;
-  const binding = database.binding;
-  return validateWorkspaceDescriptor({
-    ...descriptor,
-    dwh: {
-      engine: "postgres",
-      database: database.databaseName,
-      schema: database.schema,
-      ...(binding.port === undefined ? {} : { port: binding.port }),
-      supported_transports: [binding.transport],
-    },
-    ...(binding.transport === "rest_api" ? {
-      diagnostics: {
-        dwh_rest: {
-          method: "GET",
-          path: binding.restPath ?? "/health",
-          auth: binding.restAuth ?? "bearer",
-          response: { database: "database", schema: "schema" },
-        },
-      },
-    } : {}),
-  });
-}
-
-function setIfDefined(
-  environment: NodeJS.ProcessEnv,
-  name: string | undefined,
-  value: string | number | undefined,
-): void {
-  if (name !== undefined && value !== undefined && value !== "") environment[name] = String(value);
-}
-
-/**
- * Project one PostgreSQL Catalog row into the legacy-shaped configuration consumed by the
- * Python runtime. The authored workspace remains database-free; this object exists only for
- * the lifetime of a backend-owned runtime lease.
- */
-export function resolveCatalogRuntimeBinding(options: {
-  workspace: WorkspaceDescriptor;
-  database: WorkspaceDatabase;
-  environment: NodeJS.ProcessEnv;
-  secretRoots: readonly string[];
-  secretStore: WorkspaceSecretStore;
-}): CatalogRuntimeBindingLease {
-  const workspace = runtimeWorkspace(options.workspace, options.database);
-  const evidenceRequirements = discoverWorkspaceSecretRequirements(
-    options.workspace,
-    options.environment,
-  ).filter(({ connector }) => connector === "evidence");
-  const catalogSecretIds = Object.values(CATALOG_SECRET_IDS);
-  let materialization: WorkspaceSecretMaterialization | undefined;
-  try {
-    materialization = options.secretStore.materialize(
-      options.database.workspaceId,
-      [...catalogSecretIds, ...evidenceRequirements.map(({ id }) => id)],
-    );
-    const environment: NodeJS.ProcessEnv = { ...options.environment };
-    const roots = new Set(options.secretRoots);
-    for (const path of materialization.files.values()) roots.add(dirname(path));
-
-    const variables = buildInstallationContract(workspace).variables;
-    const variable = (role: "DWH" | "EVIDENCE", suffix: InstallationSuffix) => (
-      variables.find((candidate) => candidate.role === role && candidate.suffix === suffix)?.name
-    );
-    const binding = options.database.binding;
-    setIfDefined(environment, variable("DWH", "TRANSPORT"), binding.transport);
-    setIfDefined(environment, variable("DWH", "HOST"), binding.host);
-    setIfDefined(environment, variable("DWH", "PORT"), binding.port);
-    setIfDefined(environment, variable("DWH", "BASE_URL"), binding.baseUrl);
-    setIfDefined(environment, variable("DWH", "USER"), binding.username);
-    setIfDefined(environment, variable("DWH", "SSH_HOST"), binding.sshHost);
-    setIfDefined(environment, variable("DWH", "SSH_PORT"), binding.sshPort);
-    setIfDefined(environment, variable("DWH", "SSH_USER"), binding.sshUsername);
-    setIfDefined(environment, variable("DWH", "SSH_TARGET_HOST"), binding.sshTargetHost);
-    setIfDefined(environment, variable("DWH", "SSH_TARGET_PORT"), binding.sshTargetPort);
-    for (const [suffix, secretId] of Object.entries(CATALOG_SECRET_BY_SUFFIX) as Array<[
-      InstallationSuffix,
-      string,
-    ]>) {
-      setIfDefined(environment, variable("DWH", suffix), materialization.files.get(secretId));
-    }
-    for (const requirement of evidenceRequirements) {
-      setIfDefined(environment, requirement.variable, materialization.files.get(requirement.id));
-    }
-
-    const bindings = resolveRuntimeBindings(workspace, environment, [...roots]);
-    let released = false;
-    return {
-      workspace,
-      bindings,
-      release: () => {
-        if (released) return;
-        released = true;
-        materialization?.release();
-      },
-    };
-  } catch (error) {
-    materialization?.release();
-    throw error;
-  }
-}

backend/src/catalog/sensitive-data-suggester.ts

@@ -0,0 +1,254 @@
+import { z } from "zod";
+import type { MetadataGenerationModels } from "./metadata-generation-models.js";
+import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult, ModelCompletionUsage } from "./model-completer.js";
+import type {
+  CatalogColumn,
+  CatalogRepository,
+  CatalogTable,
+  SensitiveDataSuggestionScope,
+} from "./types.js";
+
+export type { SensitiveDataSuggestionScope } from "./types.js";
+
+// The helper accepts at most 64 KiB per message. Keep the same safety margin used by
+// Description Generation so UTF-8 structural metadata never reaches that hard limit.
+const MAX_USER_MESSAGE_BYTES = 60 * 1024;
+// Preserve ThothAI's proven completion granularity: small batches keep generation time and
+// structured-output accuracy predictable even when the helper byte limit would allow much more.
+const MAX_COLUMNS_PER_BATCH = 10;
+const responseSchema = z.object({
+  suggestions: z.array(z.object({
+    columnId: z.uuid(),
+    sensitive: z.boolean(),
+  }).strict()),
+}).strict();
+
+interface StructuralColumn {
+  columnId: string;
+  tableId: string;
+  table: string;
+  column: string;
+  dataType: string;
+  nullable: boolean;
+  primaryKey: boolean;
+  foreignKey: boolean;
+  version: number;
+  currentSensitive: boolean;
+}
+
+export interface SensitiveDataSuggestion {
+  columnId: string;
+  tableId: string;
+  tableName: string;
+  columnName: string;
+  version: number;
+  currentSensitive: boolean;
+  sensitive: boolean;
+}
+
+export class SensitiveDataSuggestionTargetNotFoundError extends Error {
+  constructor(readonly target: "database" | "table" | "column") {
+    super(`${target} not found`);
+    this.name = "SensitiveDataSuggestionTargetNotFoundError";
+  }
+}
+
+export class SensitiveDataSuggestionDuplicateTargetIdsError extends Error {
+  constructor() {
+    super("sensitive-data suggestion target IDs must be unique");
+    this.name = "SensitiveDataSuggestionDuplicateTargetIdsError";
+  }
+}
+
+export class SensitiveDataSuggestionNoEligibleColumnsError extends Error {
+  constructor(readonly scope: SensitiveDataSuggestionScope) {
+    super("selected scope has no catalog columns");
+    this.name = "SensitiveDataSuggestionNoEligibleColumnsError";
+  }
+}
+
+export class SensitiveDataSuggestionPayloadTooLargeError extends Error {
+  constructor() {
+    super("sensitive-data suggestion structural metadata is too large");
+    this.name = "SensitiveDataSuggestionPayloadTooLargeError";
+  }
+}
+
+export class SensitiveDataSuggestionInvalidResponseError extends Error {
+  constructor() {
+    super("sensitive-data suggestion response is invalid");
+    this.name = "SensitiveDataSuggestionInvalidResponseError";
+  }
+}
+
+function userContent(
+  database: { databaseName: string; schema: string },
+  columns: readonly StructuralColumn[],
+): string {
+  return JSON.stringify({
+    database: database.databaseName,
+    schema: database.schema,
+    columns: columns.map((column) => ({
+      columnId: column.columnId,
+      table: column.table,
+      column: column.column,
+      dataType: column.dataType,
+      nullable: column.nullable,
+      primaryKey: column.primaryKey,
+      foreignKey: column.foreignKey,
+    })),
+  });
+}
+
+function batchesFor(
+  database: { databaseName: string; schema: string },
+  columns: readonly StructuralColumn[],
+): StructuralColumn[][] {
+  const batches: StructuralColumn[][] = [];
+  let current: StructuralColumn[] = [];
+  for (const column of columns) {
+    if (current.length === MAX_COLUMNS_PER_BATCH) {
+      batches.push(current);
+      current = [];
+    }
+    const candidate = [...current, column];
+    if (Buffer.byteLength(userContent(database, candidate), "utf8") <= MAX_USER_MESSAGE_BYTES) {
+      current = candidate;
+      continue;
+    }
+    if (current.length === 0) throw new SensitiveDataSuggestionPayloadTooLargeError();
+    batches.push(current);
+    current = [column];
+    if (Buffer.byteLength(userContent(database, current), "utf8") > MAX_USER_MESSAGE_BYTES) {
+      throw new SensitiveDataSuggestionPayloadTooLargeError();
+    }
+  }
+  if (current.length > 0) batches.push(current);
+  return batches;
+}
+
+function structuralColumn(table: CatalogTable, column: CatalogColumn): StructuralColumn {
+  return {
+    columnId: column.id,
+    tableId: table.id,
+    table: table.name,
+    column: column.name,
+    dataType: column.dataType,
+    nullable: column.isNullable,
+    primaryKey: column.isPrimaryKey,
+    foreignKey: column.isForeignKey,
+    version: column.version,
+    currentSensitive: column.sensitive,
+  };
+}
+
+const systemMessage: ModelCompletionMessage = {
+  role: "system",
+  content: [
+    "Classify whether each database column is likely to contain sensitive source values.",
+    "Use only the supplied structural metadata. Return strict JSON with this exact shape:",
+    '{"suggestions":[{"columnId":"uuid","sensitive":true}]}',
+    "Return every supplied column exactly once. Do not add explanations or markdown.",
+  ].join("\n"),
+};
+
+export class SensitiveDataSuggester {
+  constructor(
+    private readonly repository: CatalogRepository,
+    private readonly models: MetadataGenerationModels,
+    private readonly completer: ModelCompleter,
+  ) {}
+
+  private async selectColumns(
+    databaseId: string,
+    scope: SensitiveDataSuggestionScope,
+    targetIds: readonly string[],
+  ): Promise<StructuralColumn[]> {
+    if (new Set(targetIds).size !== targetIds.length) {
+      throw new SensitiveDataSuggestionDuplicateTargetIdsError();
+    }
+    const tables = await this.repository.listTables(databaseId);
+    const tableIds = new Set(targetIds);
+    const selectedTables = scope === "selected_tables"
+      ? tables.filter((table) => tableIds.has(table.id))
+      : tables;
+    if (scope === "selected_tables" && selectedTables.length !== targetIds.length) {
+      throw new SensitiveDataSuggestionTargetNotFoundError("table");
+    }
+
+    const columns = (await Promise.all(selectedTables.map(async (table) => (
+      (await this.repository.listColumns(databaseId, table.id)).map((column) => (
+        structuralColumn(table, column)
+      ))
+    )))).flat();
+    const columnIds = new Set(targetIds);
+    const selectedColumns = scope === "selected_columns"
+      ? columns.filter((column) => columnIds.has(column.columnId))
+      : columns;
+    if (scope === "selected_columns" && selectedColumns.length !== targetIds.length) {
+      throw new SensitiveDataSuggestionTargetNotFoundError("column");
+    }
+    if (selectedColumns.length === 0) {
+      throw new SensitiveDataSuggestionNoEligibleColumnsError(scope);
+    }
+    return selectedColumns;
+  }
+
+  async suggest(
+    databaseId: string,
+    modelId: string,
+    scope: SensitiveDataSuggestionScope,
+    targetIds: readonly string[],
+    signal: AbortSignal,
+    onPrepared?: (total: number) => void | Promise<void>,
+    onProgress?: (processed: number, suggestions: readonly SensitiveDataSuggestion[]) => void | Promise<void>,
+    onUsage?: (usage: ModelCompletionUsage) => void | Promise<void>,
+  ): Promise<readonly SensitiveDataSuggestion[]> {
+    const database = await this.repository.get(databaseId);
+    if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
+    const columns = await this.selectColumns(databaseId, scope, targetIds);
+    await onPrepared?.(columns.length);
+    const model = this.models.resolve(modelId);
+    const suggestions: SensitiveDataSuggestion[] = [];
+
+    for (const batch of batchesFor(database, columns)) {
+      let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
+      for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
+        const completion = await this.completer.complete({
+          model,
+          signal,
+          messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
+        });
+        const result: ModelCompletionResult = typeof completion === "string"
+          ? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
+          : completion;
+        await onUsage?.(result.usage);
+        const content = result.content;
+        try {
+          const parsed = responseSchema.parse(JSON.parse(content));
+          const expected = new Set(batch.map((column) => column.columnId));
+          const candidate = new Map(parsed.suggestions.map((suggestion) => [suggestion.columnId, suggestion]));
+          if (candidate.size !== parsed.suggestions.length
+            || candidate.size !== expected.size
+            || [...candidate.keys()].some((columnId) => !expected.has(columnId))) {
+            throw new SensitiveDataSuggestionInvalidResponseError();
+          }
+          received = candidate;
+        } catch {
+          if (attempt === 1) throw new SensitiveDataSuggestionInvalidResponseError();
+        }
+      }
+      suggestions.push(...batch.map((column) => ({
+        columnId: column.columnId,
+        tableId: column.tableId,
+        tableName: column.table,
+        columnName: column.column,
+        version: column.version,
+        currentSensitive: column.currentSensitive,
+        sensitive: received!.get(column.columnId)!.sensitive,
+      })));
+      await onProgress?.(suggestions.length, suggestions.slice(-batch.length));
+    }
+    return suggestions;
+  }
+}

backend/src/catalog/sensitive-data-suggestion-runner.ts

@@ -0,0 +1,136 @@
+import type {
+  SensitiveDataSuggestion,
+} from "./sensitive-data-suggester.js";
+import {
+  SensitiveDataSuggester,
+  SensitiveDataSuggestionTargetNotFoundError,
+} from "./sensitive-data-suggester.js";
+import type {
+  CatalogRepository,
+  SensitiveDataSuggestionRun,
+  SensitiveDataSuggestionScope,
+} from "./types.js";
+import type { ModelCompletionUsage } from "./model-completer.js";
+
+const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
+const failedMessage = "Sensitive-field suggestion generation failed.";
+
+export interface SensitiveDataSuggestionRunResult {
+  suggestions: readonly SensitiveDataSuggestion[];
+  run: SensitiveDataSuggestionRun;
+}
+
+export class SensitiveDataSuggestionRunner {
+  constructor(
+    private readonly repository: CatalogRepository,
+    private readonly suggester: SensitiveDataSuggester,
+  ) {}
+
+  async initialize(): Promise<void> {
+    if (!(await this.repository.available())) return;
+    const interrupted = await this.repository.interruptActiveSensitiveDataSuggestionRuns(
+      interruptedMessage,
+    );
+    for (const run of interrupted) {
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        run.id,
+        "warning",
+        interruptedMessage,
+      );
+    }
+  }
+
+  async run(
+    databaseId: string,
+    modelId: string,
+    scope: SensitiveDataSuggestionScope,
+    targetIds: readonly string[],
+    signal: AbortSignal,
+  ): Promise<SensitiveDataSuggestionRunResult> {
+    if (!(await this.repository.get(databaseId))) {
+      throw new SensitiveDataSuggestionTargetNotFoundError("database");
+    }
+    const started = await this.repository.createSensitiveDataSuggestionRun(
+      databaseId,
+      scope,
+      modelId,
+    );
+
+    try {
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        started.id,
+        "info",
+        "Sensitive-field suggestion generation started.",
+      );
+      const suggestions = await this.suggester.suggest(
+        databaseId,
+        modelId,
+        scope,
+        targetIds,
+        signal,
+        async (total) => {
+          const prepared = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+            total,
+          });
+          if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
+        },
+        async (processed, batch) => {
+          const suggestedSensitive = batch.filter((suggestion) => suggestion.sensitive).length;
+          const suggestedNonSensitive = batch.length - suggestedSensitive;
+          const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
+          if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
+          const progress = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+            suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
+            suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
+          });
+          if (!progress) throw new Error("Sensitive Data Suggestion Run disappeared");
+          await this.repository.appendSensitiveDataSuggestionEvent(
+            started.id,
+            "info",
+            `Classified ${processed} of ${progress.total} columns.`,
+          );
+        },
+        async (usage: ModelCompletionUsage) => {
+          const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
+          if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
+          await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+            inputTokens: current.inputTokens + usage.input,
+            cacheReadTokens: current.cacheReadTokens + usage.cacheRead,
+            outputTokens: current.outputTokens + usage.output,
+          });
+        },
+      );
+      const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
+      const suggestedNonSensitive = suggestions.length - suggestedSensitive;
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        started.id,
+        "info",
+        `Sensitive-field suggestion generation completed for ${suggestions.length} column${
+          suggestions.length === 1 ? "" : "s"
+        }.`,
+      );
+      const completed = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+        status: "completed",
+        total: suggestions.length,
+        suggestedSensitive,
+        suggestedNonSensitive,
+        finishedAt: new Date().toISOString(),
+        errorSummary: null,
+      });
+      if (!completed) throw new Error("Sensitive Data Suggestion Run disappeared");
+      return { suggestions, run: completed };
+    } catch (error) {
+      await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+        status: "failed",
+        finishedAt: new Date().toISOString(),
+        errorSummary: failedMessage,
+      }).catch(() => undefined);
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        started.id,
+        "error",
+        failedMessage,
+      ).catch(() => undefined);
+      throw error;
+    }
+  }
+}

backend/src/catalog/sensitivity-analysis-runner.ts

@@ -1,178 +0,0 @@
-import type {
-  SensitivityReviewItem,
-} from "./sensitivity-analysis-service.js";
-import {
-  SENSITIVITY_POLICY_VERSION,
-  SensitivityAnalysisInterruptedError,
-  SensitivityAnalysisService,
-  SensitivityAnalysisTargetNotFoundError,
-} from "./sensitivity-analysis-service.js";
-import type {
-  CatalogRepository,
-  SensitivityAnalysisRun,
-  SensitivityAnalysisScope,
-} from "./types.js";
-
-const interruptedMessage = "Local sensitivity analysis was interrupted by backend restart.";
-const interruptedDuringRunMessage = "Local sensitivity analysis was interrupted before completion.";
-const failedMessage = "Local sensitivity analysis failed.";
-
-function ensureActive(signal: AbortSignal): void {
-  if (signal.aborted) throw new SensitivityAnalysisInterruptedError();
-}
-
-export interface SensitivityAnalysisRunResult {
-  suggestions: readonly SensitivityReviewItem[];
-  run: SensitivityAnalysisRun;
-}
-
-export class SensitivityAnalysisRunner {
-  constructor(
-    private readonly repository: CatalogRepository,
-    private readonly analysis: SensitivityAnalysisService,
-  ) {}
-
-  async initialize(): Promise<void> {
-    if (!(await this.repository.available())) return;
-    const interrupted = await this.repository.interruptActiveSensitivityAnalysisRuns(
-      interruptedMessage,
-    );
-    for (const run of interrupted) {
-      await this.repository.appendSensitivityAnalysisEvent(
-        run.id,
-        "warning",
-        interruptedMessage,
-      );
-    }
-  }
-
-  async run(
-    databaseId: string,
-    scope: SensitivityAnalysisScope,
-    targetIds: readonly string[],
-    signal: AbortSignal,
-  ): Promise<SensitivityAnalysisRunResult> {
-    ensureActive(signal);
-    const database = await this.repository.get(databaseId);
-    ensureActive(signal);
-    if (!database) {
-      throw new SensitivityAnalysisTargetNotFoundError("database");
-    }
-    ensureActive(signal);
-    const started = await this.repository.createSensitivityAnalysisRun(
-      databaseId,
-      scope,
-      { engine: "local", policyVersion: SENSITIVITY_POLICY_VERSION },
-    );
-    let preparedTotal = 0;
-    let processedSensitive = 0;
-    let processedNonSensitive = 0;
-
-    try {
-      ensureActive(signal);
-      await this.repository.appendSensitivityAnalysisEvent(
-        started.id,
-        "info",
-        "Local sensitivity analysis started.",
-      );
-      ensureActive(signal);
-      const suggestions = await this.analysis.analyze(
-        databaseId,
-        scope,
-        targetIds,
-        signal,
-        async (total) => {
-          ensureActive(signal);
-          preparedTotal = total;
-          const prepared = await this.repository.updateSensitivityAnalysisRun(started.id, {
-            total,
-          });
-          ensureActive(signal);
-          if (!prepared) throw new Error("Sensitivity Analysis Run disappeared");
-        },
-        async (processed, batch) => {
-          ensureActive(signal);
-          const suggestedSensitive = batch.filter(
-            (suggestion) => suggestion.assessment === "sensitive",
-          ).length;
-          const suggestedNonSensitive = batch.filter(
-            (suggestion) => suggestion.assessment === "non_sensitive",
-          ).length;
-          const current = await this.repository.getSensitivityAnalysisRun(started.id);
-          ensureActive(signal);
-          if (!current) throw new Error("Sensitivity Analysis Run disappeared");
-          const progress = await this.repository.updateSensitivityAnalysisRun(started.id, {
-            suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
-            suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
-          });
-          if (!progress) throw new Error("Sensitivity Analysis Run disappeared");
-          processedSensitive += suggestedSensitive;
-          processedNonSensitive += suggestedNonSensitive;
-          ensureActive(signal);
-          await this.repository.appendSensitivityAnalysisEvent(
-            started.id,
-            "info",
-            `Assessed ${processed} of ${progress.total} columns locally.`,
-          );
-          ensureActive(signal);
-        },
-        async (message) => {
-          ensureActive(signal);
-          await this.repository.appendSensitivityAnalysisEvent(
-            started.id,
-            "info",
-            message,
-          );
-          ensureActive(signal);
-        },
-      );
-      ensureActive(signal);
-      const suggestedSensitive = suggestions.filter(
-        (suggestion) => suggestion.assessment === "sensitive",
-      ).length;
-      const suggestedNonSensitive = suggestions.filter(
-        (suggestion) => suggestion.assessment === "non_sensitive",
-      ).length;
-      await this.repository.appendSensitivityAnalysisEvent(
-        started.id,
-        "info",
-        `Local sensitivity analysis completed for ${suggestions.length} column${
-          suggestions.length === 1 ? "" : "s"
-        }.`,
-      );
-      ensureActive(signal);
-      const completed = await this.repository.updateSensitivityAnalysisRun(started.id, {
-        status: "completed",
-        total: suggestions.length,
-        suggestedSensitive,
-        suggestedNonSensitive,
-        unknown: 0,
-        finishedAt: new Date().toISOString(),
-        errorSummary: null,
-      });
-      ensureActive(signal);
-      if (!completed) throw new Error("Sensitivity Analysis Run disappeared");
-      return { suggestions, run: completed };
-    } catch (error) {
-      const interrupted = signal.aborted || error instanceof SensitivityAnalysisInterruptedError;
-      const message = interrupted ? interruptedDuringRunMessage : failedMessage;
-      await this.repository.updateSensitivityAnalysisRun(started.id, {
-        status: interrupted ? "interrupted" : "failed",
-        ...(interrupted ? {
-          total: preparedTotal,
-          suggestedSensitive: processedSensitive,
-          suggestedNonSensitive: processedNonSensitive,
-          unknown: Math.max(0, preparedTotal - processedSensitive - processedNonSensitive),
-        } : {}),
-        finishedAt: new Date().toISOString(),
-        errorSummary: message,
-      }).catch(() => undefined);
-      await this.repository.appendSensitivityAnalysisEvent(
-        started.id,
-        interrupted ? "warning" : "error",
-        message,
-      ).catch(() => undefined);
-      throw error;
-    }
-  }
-}

backend/src/catalog/sensitivity-analysis-service.ts

@@ -1,184 +0,0 @@
-import type {
-  SensitivityClassifier,
-  SensitivityColumnAssessment,
-  SensitivityEvidence,
-  SensitivityNerBudget,
-} from "./sensitivity-classifier.js";
-import type {
-  CatalogColumn,
-  CatalogRepository,
-  CatalogTable,
-  SensitivityAnalysisScope,
-} from "./types.js";
-
-export type { SensitivityAnalysisScope } from "./types.js";
-export const SENSITIVITY_POLICY_VERSION = "sensitivity-v4";
-
-interface SelectedColumn {
-  table: CatalogTable;
-  column: CatalogColumn;
-}
-
-export interface SensitivityReviewItem {
-  columnId: string;
-  tableId: string;
-  tableName: string;
-  columnName: string;
-  version: number;
-  currentSensitive: boolean;
-  sensitive: boolean;
-  assessment: SensitivityColumnAssessment["assessment"];
-  evidence: readonly SensitivityEvidence[];
-  observedValues: number;
-  coverage: SensitivityColumnAssessment["coverage"];
-}
-
-export class SensitivityAnalysisTargetNotFoundError extends Error {
-  constructor(readonly target: "database" | "table" | "column") {
-    super(`${target} not found`);
-    this.name = "SensitivityAnalysisTargetNotFoundError";
-  }
-}
-
-export class SensitivityAnalysisDuplicateTargetIdsError extends Error {
-  constructor() {
-    super("sensitivity analysis target IDs must be unique");
-    this.name = "SensitivityAnalysisDuplicateTargetIdsError";
-  }
-}
-
-export class SensitivityAnalysisInterruptedError extends Error {
-  constructor() {
-    super("sensitivity analysis interrupted");
-    this.name = "SensitivityAnalysisInterruptedError";
-  }
-}
-
-function ensureActive(signal: AbortSignal): void {
-  if (signal.aborted) throw new SensitivityAnalysisInterruptedError();
-}
-
-export class SensitivityAnalysisNoEligibleColumnsError extends Error {
-  constructor(readonly scope: SensitivityAnalysisScope) {
-    super("selected scope has no catalog columns");
-    this.name = "SensitivityAnalysisNoEligibleColumnsError";
-  }
-}
-
-/** Selection and table orchestration around the single SensitivityClassifier decision module. */
-export class SensitivityAnalysisService {
-  constructor(
-    private readonly repository: CatalogRepository,
-    private readonly classifier: SensitivityClassifier,
-    private readonly options: { nerBudgetMs?: number } = {},
-  ) {}
-
-  private async selectColumns(
-    databaseId: string,
-    scope: SensitivityAnalysisScope,
-    targetIds: readonly string[],
-    signal: AbortSignal,
-  ): Promise<readonly SelectedColumn[]> {
-    ensureActive(signal);
-    if (new Set(targetIds).size !== targetIds.length) {
-      throw new SensitivityAnalysisDuplicateTargetIdsError();
-    }
-    const tables = await this.repository.listTables(databaseId);
-    ensureActive(signal);
-    const tableIds = new Set(targetIds);
-    const selectedTables = scope === "selected_tables"
-      ? tables.filter((table) => tableIds.has(table.id))
-      : tables;
-    if (scope === "selected_tables" && selectedTables.length !== targetIds.length) {
-      throw new SensitivityAnalysisTargetNotFoundError("table");
-    }
-    const columns = (await Promise.all(selectedTables.map(async (table) => (
-      (await this.repository.listColumns(databaseId, table.id)).map((column) => ({ table, column }))
-    )))).flat();
-    ensureActive(signal);
-    const columnIds = new Set(targetIds);
-    const selectedColumns = scope === "selected_columns"
-      ? columns.filter(({ column }) => columnIds.has(column.id))
-      : columns;
-    if (scope === "selected_columns" && selectedColumns.length !== targetIds.length) {
-      throw new SensitivityAnalysisTargetNotFoundError("column");
-    }
-    if (selectedColumns.length === 0) {
-      throw new SensitivityAnalysisNoEligibleColumnsError(scope);
-    }
-    return selectedColumns;
-  }
-
-  async analyze(
-    databaseId: string,
-    scope: SensitivityAnalysisScope,
-    targetIds: readonly string[],
-    signal: AbortSignal,
-    onPrepared?: (total: number) => void | Promise<void>,
-    onProgress?: (processed: number, suggestions: readonly SensitivityReviewItem[]) => void | Promise<void>,
-    onActivity?: (message: string) => void | Promise<void>,
-  ): Promise<readonly SensitivityReviewItem[]> {
-    const configuredNerBudget = this.options.nerBudgetMs ?? 10_000;
-    const nerBudget: SensitivityNerBudget = {
-      remainingMs: Number.isFinite(configuredNerBudget) && configuredNerBudget >= 0
-        ? configuredNerBudget
-        : 10_000,
-    };
-    ensureActive(signal);
-    const database = await this.repository.get(databaseId);
-    ensureActive(signal);
-    if (!database) throw new SensitivityAnalysisTargetNotFoundError("database");
-    const selected = await this.selectColumns(databaseId, scope, targetIds, signal);
-    await onPrepared?.(selected.length);
-    ensureActive(signal);
-    const byTable = new Map<string, SelectedColumn[]>();
-    for (const item of selected) {
-      const items = byTable.get(item.table.id) ?? [];
-      items.push(item);
-      byTable.set(item.table.id, items);
-    }
-    const tableTargets = [...byTable.values()].map((items) => {
-      const first = items[0]!;
-      return {
-        database,
-        table: first.table,
-        columns: items.map(({ column }) => column),
-      };
-    });
-    const assessments = await this.classifier.assess(
-      tableTargets,
-      signal,
-      nerBudget,
-      onActivity,
-    );
-    ensureActive(signal);
-    const assessmentById = new Map(assessments.map((assessment) => [
-      assessment.columnId,
-      assessment,
-    ]));
-    const suggestions: SensitivityReviewItem[] = [];
-    for (const items of byTable.values()) {
-      ensureActive(signal);
-      const batch = items.map(({ table, column }) => {
-        const assessment = assessmentById.get(column.id)!;
-        return {
-          columnId: column.id,
-          tableId: table.id,
-          tableName: table.name,
-          columnName: column.name,
-          version: column.version,
-          currentSensitive: column.sensitive,
-          sensitive: assessment.proposedSensitive,
-          assessment: assessment.assessment,
-          evidence: assessment.evidence,
-          observedValues: assessment.observedValues,
-          coverage: assessment.coverage,
-        };
-      });
-      suggestions.push(...batch);
-      await onProgress?.(suggestions.length, batch);
-      ensureActive(signal);
-    }
-    return suggestions;
-  }
-}

backend/src/catalog/sensitivity-classifier.ts

@@ -1,535 +0,0 @@
-import type { CatalogColumn, CatalogTable, WorkspaceDatabase } from "./types.js";
-import { findPhoneNumbersInText } from "libphonenumber-js/max";
-import validator from "validator";
-
-export type SensitivityAssessment = "sensitive" | "non_sensitive";
-
-export interface SensitivityEvidence {
-  kind: "metadata" | "content" | "length" | "ner" | "coverage" | "type";
-  ruleId: string;
-  label?: string;
-  confidence?: number;
-}
-
-export interface SensitivityValueObservation {
-  columnId: string;
-  value: string | null;
-  characterLength: number | null;
-}
-
-export interface SensitivityScanCoverage {
-  kind: "complete" | "sampled";
-  observedValues: number;
-}
-
-export interface SensitivityTableScan {
-  batches: readonly (readonly SensitivityValueObservation[])[];
-  coverage: SensitivityScanCoverage;
-}
-
-export interface SensitivityScanRequest {
-  database: WorkspaceDatabase;
-  table: CatalogTable;
-  columns: readonly CatalogColumn[];
-  valuesPerColumn: number;
-  sampleOffset: number;
-  sampleSeed: number;
-  queryTimeoutMs: number;
-  fullScanThreshold?: number;
-}
-
-export interface SensitivityValueSource {
-  scanTable(
-    request: SensitivityScanRequest,
-    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
-    signal: AbortSignal,
-  ): Promise<SensitivityScanCoverage>;
-}
-
-export interface LocalNerCandidate {
-  columnId: string;
-  text: string;
-}
-
-export interface LocalNerEvidence {
-  columnId: string;
-  label: string;
-  confidence: number;
-}
-
-export interface SensitivityNerBudget {
-  remainingMs: number;
-}
-
-/** Optional local detector. It returns evidence only; it never decides a column assessment. */
-export interface LocalNerDetector {
-  warmup?(): Promise<void>;
-  isReady?(): boolean;
-  detect(
-    candidates: readonly LocalNerCandidate[],
-    signal: AbortSignal,
-    deadline: number,
-  ): Promise<readonly LocalNerEvidence[]>;
-  close?(): Promise<void>;
-}
-
-export interface SensitivityColumnAssessment {
-  columnId: string;
-  assessment: SensitivityAssessment;
-  proposedSensitive: boolean;
-  evidence: readonly SensitivityEvidence[];
-  observedValues: number;
-  coverage: "metadata" | "complete" | "sampled" | "no_values";
-}
-
-export interface SensitivityTableTarget {
-  database: WorkspaceDatabase;
-  table: CatalogTable;
-  columns: readonly CatalogColumn[];
-}
-
-const EMAIL = /(?<![\p{L}\p{N}._%+-])[\p{L}\p{N}._%+-]+@[\p{L}\p{N}.-]+\.[\p{L}]{2,63}(?![\p{L}\p{N}._%+-])/giu;
-const DIRECT_IDENTIFIER_NAMES = new Set([
-  "address", "birth_date", "codice_fiscale", "date_of_birth", "dob", "email", "e_mail",
-  "bic", "first_name", "fiscal_code", "full_name", "iban", "indirizzo", "last_name", "mobile",
-  "nome", "passport", "phone", "surname", "swift", "swift_code", "tax_id", "telefono",
-]);
-const CREDENTIAL_NAME = /(?:^|_)(?:api_key|credential|password|passwd|private_key|pwd|secret|token)(?:_|$)/u;
-const HEALTH_NAME = /(?:^|_)(?:anamnesi|clinical|diagnos(?:i|is)|health|medical|patient|patologia|therapy|terapia)(?:_|$)/u;
-const CLINICAL_TERM = /(?:^|[^\p{L}])(?:allergi[ae]|anamnesi|carcinoma|chemioterapia|diabete|diagnos[ei]|epatite|farmac[io]|gravidanza|hiv|metastasi|neoplasia|patologia|radioterapia|referto|terapia|tumore)(?:$|[^\p{L}])/iu;
-const UNSUPPORTED_BINARY_TYPE = /(?:^|\s)(?:binary|blob|bytea|image|varbinary)(?:\s|$|\()/iu;
-const DEEP_TEXT_TYPE = /(?:^|\s)(?:char|character|citext|clob|json|jsonb|nchar|nvarchar|string|text|varchar|xml)(?:\s|$|\()/iu;
-const MAX_NER_CANDIDATES_PER_REQUEST = 128;
-const MAX_CONCURRENT_TABLE_SCANS = 2;
-
-export const SENSITIVITY_SAMPLE_PHASES = [
-  { targetValuesPerColumn: 300, additionalValuesPerColumn: 300, sampleSeed: 37, deepTextOnly: false },
-  { targetValuesPerColumn: 1_000, additionalValuesPerColumn: 700, sampleSeed: 73, deepTextOnly: false },
-  { targetValuesPerColumn: 3_000, additionalValuesPerColumn: 2_000, sampleSeed: 109, deepTextOnly: true },
-] as const;
-
-function normalizedName(value: string): string {
-  return value.normalize("NFKD")
-    .replace(/[\u0300-\u036f]/g, "")
-    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
-    .toLocaleLowerCase("en-US")
-    .replace(/[^a-z0-9]+/g, "_")
-    .replace(/^_+|_+$/g, "");
-}
-
-function boundedCount(value: number | undefined, fallback: number, maximum: number): number {
-  return value === undefined || !Number.isSafeInteger(value)
-    ? fallback
-    : Math.max(1, Math.min(value, maximum));
-}
-
-function metadataEvidence(column: CatalogColumn): SensitivityEvidence | undefined {
-  const ruleId = sensitiveNameRule(column.name);
-  return ruleId ? { kind: "metadata", ruleId } : undefined;
-}
-
-function nonSensitiveStructuralEvidence(column: CatalogColumn): SensitivityEvidence | undefined {
-  if (column.dataType.trim().toLowerCase() !== "bigint") return undefined;
-  if (column.isPrimaryKey || column.primaryKeyPosition !== null) {
-    return {
-      kind: "type",
-      ruleId: "type.bigint_primary_key_non_informative",
-      label: "non-informative bigint primary key",
-    };
-  }
-  if (normalizedName(column.name) === "pk") {
-    return {
-      kind: "metadata",
-      ruleId: "metadata.bigint_pk_identifier_non_informative",
-      label: "non-informative conventional bigint primary-key identifier",
-    };
-  }
-  return undefined;
-}
-
-function sensitiveNameRule(value: string): string | undefined {
-  const name = normalizedName(value);
-  if (DIRECT_IDENTIFIER_NAMES.has(name)) {
-    return "metadata.direct_identifier";
-  }
-  if (CREDENTIAL_NAME.test(name)) {
-    return "metadata.credential";
-  }
-  if (HEALTH_NAME.test(name)) {
-    return "metadata.health";
-  }
-  return undefined;
-}
-
-const ITALIAN_FISCAL_CODE = /(?<![A-Z0-9])[A-Z]{6}[0-9LMNPQRSTUV]{2}[ABCDEHLMPRST][0-9LMNPQRSTUV]{2}[A-Z][0-9LMNPQRSTUV]{3}[A-Z](?![A-Z0-9])/giu;
-const FISCAL_ODD: Record<string, number> = {
-  "0": 1, "1": 0, "2": 5, "3": 7, "4": 9, "5": 13, "6": 15, "7": 17, "8": 19, "9": 21,
-  A: 1, B: 0, C: 5, D: 7, E: 9, F: 13, G: 15, H: 17, I: 19, J: 21,
-  K: 2, L: 4, M: 18, N: 20, O: 11, P: 3, Q: 6, R: 8, S: 12, T: 14,
-  U: 16, V: 10, W: 22, X: 25, Y: 24, Z: 23,
-};
-
-function validItalianFiscalCode(candidate: string): boolean {
-  const value = candidate.toUpperCase();
-  if (value.length !== 16) return false;
-  let sum = 0;
-  for (let index = 0; index < 15; index += 1) {
-    const character = value[index]!;
-    if (index % 2 === 0) sum += FISCAL_ODD[character] ?? -1000;
-    else sum += /\d/u.test(character) ? Number(character) : character.charCodeAt(0) - 65;
-  }
-  return String.fromCharCode(65 + (sum % 26)) === value[15];
-}
-
-function validIban(candidate: string): boolean {
-  const value = candidate.replace(/\s+/gu, "").toUpperCase();
-  if (!/^[A-Z]{2}\d{2}[A-Z0-9]{11,30}$/u.test(value)) return false;
-  const rearranged = value.slice(4) + value.slice(0, 4);
-  let remainder = 0;
-  for (const character of rearranged) {
-    const digits = /\d/u.test(character) ? character : String(character.charCodeAt(0) - 55);
-    for (const digit of digits) remainder = (remainder * 10 + Number(digit)) % 97;
-  }
-  return remainder === 1;
-}
-
-function validPaymentCard(candidate: string): boolean {
-  const digits = candidate.replace(/[ -]/gu, "");
-  if (!/^\d{13,19}$/u.test(digits) || /^(\d)\1+$/u.test(digits)) return false;
-  let sum = 0;
-  let double = false;
-  for (let index = digits.length - 1; index >= 0; index -= 1) {
-    let digit = Number(digits[index]);
-    if (double) {
-      digit *= 2;
-      if (digit > 9) digit -= 9;
-    }
-    sum += digit;
-    double = !double;
-  }
-  return sum % 10 === 0;
-}
-
-function jsonHasSensitiveKey(value: string): boolean {
-  const trimmed = value.trim();
-  if (!(trimmed.startsWith("{") || trimmed.startsWith("["))) return false;
-  try {
-    const pending: Array<{ value: unknown; depth: number }> = [{ value: JSON.parse(trimmed), depth: 0 }];
-    let visited = 0;
-    while (pending.length > 0 && visited < 1_000) {
-      const item = pending.pop()!;
-      visited += 1;
-      if (item.depth > 8 || item.value === null || typeof item.value !== "object") continue;
-      if (Array.isArray(item.value)) {
-        for (const child of item.value) pending.push({ value: child, depth: item.depth + 1 });
-        continue;
-      }
-      for (const [key, child] of Object.entries(item.value)) {
-        if (sensitiveNameRule(key)) return true;
-        pending.push({ value: child, depth: item.depth + 1 });
-      }
-    }
-  } catch {
-    return false;
-  }
-  return false;
-}
-
-function contentEvidence(value: string): SensitivityEvidence | undefined {
-  if (/-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----/u.test(value)) {
-    return { kind: "content", ruleId: "credential.private_key" };
-  }
-  if (/(?:^|[^A-Z0-9])AKIA[A-Z0-9]{16}(?![A-Z0-9])/u.test(value)
-    || /(?:^|[^A-Za-z0-9_])gh[pousr]_[A-Za-z0-9_]{30,}(?![A-Za-z0-9_])/u.test(value)
-    || /(?:^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}(?![A-Za-z0-9_-])/u.test(value)) {
-    return { kind: "content", ruleId: "credential.access_key" };
-  }
-  if (/(?:^|[^\p{L}\p{N}_])(?:api[_ -]?key|access[_ -]?token|password|passwd|pwd|secret)\s*[:=]\s*[^\s,;]{4,}/iu.test(value)) {
-    return { kind: "content", ruleId: "credential.key_value" };
-  }
-  if (CLINICAL_TERM.test(value)) return { kind: "content", ruleId: "health.clinical_term" };
-  for (const match of value.matchAll(EMAIL)) {
-    if (validator.isEmail(match[0])) return { kind: "content", ruleId: "pii.email" };
-  }
-  for (const match of value.matchAll(ITALIAN_FISCAL_CODE)) {
-    if (validItalianFiscalCode(match[0])) {
-      return { kind: "content", ruleId: "pii.italian_fiscal_code" };
-    }
-  }
-  for (const match of value.matchAll(/\b(?:passaporto|passport)(?:\s+(?:numero|number|n\.?))?\s*[:#-]?\s*([A-Z0-9]{9})\b/giu)) {
-    if (validator.isPassportNumber(match[1]!, "IT")) {
-      return { kind: "content", ruleId: "pii.passport_number" };
-    }
-  }
-  for (const match of value.matchAll(/\bC[A-Z]\d{5}[A-Z]{2}\b/giu)) {
-    if (validator.isIdentityCard(match[0], "IT")) {
-      return { kind: "content", ruleId: "pii.identity_card" };
-    }
-  }
-  if (/\b(?:patente(?:\s+di\s+guida)?|driving\s+licen[cs]e)(?:\s+(?:numero|number|n\.?))?\s*[:#-]?\s*[A-Z0-9]{8,12}\b/iu.test(value)) {
-    return { kind: "content", ruleId: "pii.drivers_license_number" };
-  }
-  for (const match of value.matchAll(/(?<![A-Z0-9])[A-Z]{2}\d{2}(?:\s?[A-Z0-9]){11,30}(?![A-Z0-9])/giu)) {
-    if (validIban(match[0])) return { kind: "content", ruleId: "financial.iban" };
-  }
-  for (const match of value.matchAll(/(?<!\d)(?:\d[ -]?){13,19}(?!\d)/gu)) {
-    if (validPaymentCard(match[0])) {
-      return { kind: "content", ruleId: "financial.payment_card" };
-    }
-  }
-  for (const match of value.matchAll(/(?<![A-Z0-9])[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?(?![A-Z0-9])/giu)) {
-    const before = value.slice(Math.max(0, (match.index ?? 0) - 24), match.index ?? 0);
-    if (/\b(?:bic|swift)\s*[:=-]?\s*$/iu.test(before) && validator.isBIC(match[0])) {
-      return { kind: "content", ruleId: "financial.bic" };
-    }
-  }
-  for (const match of value.matchAll(/(?<!\d)(?:IT[ .-]?)?\d{11}(?!\d)/giu)) {
-    const candidate = match[0].replace(/[ .-]/gu, "");
-    if (validator.isVAT(candidate.replace(/^IT/iu, ""), "IT")) {
-      return { kind: "content", ruleId: "pii.italian_vat" };
-    }
-  }
-  for (const match of value.matchAll(/(?<![A-F0-9])(?:[A-F0-9]{2}[:-]){5}[A-F0-9]{2}(?![A-F0-9])/giu)) {
-    if (validator.isMACAddress(match[0])) {
-      return { kind: "content", ruleId: "network.mac_address" };
-    }
-  }
-  for (const match of value.matchAll(/(?<![A-F0-9:.])[A-F0-9:.]{3,45}(?![A-F0-9:.])/giu)) {
-    if (validator.isIP(match[0])) return { kind: "content", ruleId: "network.ip_address" };
-  }
-  for (const match of value.matchAll(/(?<![A-F0-9-])[0-9A-F]{8}-[0-9A-F]{4}-[1-8][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}(?![A-F0-9-])/giu)) {
-    if (validator.isUUID(match[0])) return { kind: "content", ruleId: "pii.uuid" };
-  }
-  for (const match of value.matchAll(/\b(?:https?|ftp):\/\/[^\s<>"']+/giu)) {
-    const candidate = match[0].replace(/[.,;:!?\])}]+$/u, "");
-    if (validator.isURL(candidate, { require_protocol: true })) {
-      return { kind: "content", ruleId: "network.url" };
-    }
-  }
-  if (findPhoneNumbersInText(value, "IT").some((match) => match.number.isValid())) {
-    return { kind: "content", ruleId: "pii.phone_number" };
-  }
-  if (jsonHasSensitiveKey(value)) {
-    return { kind: "content", ruleId: "pii.json_sensitive_key" };
-  }
-  return undefined;
-}
-
-interface ColumnState {
-  column: CatalogColumn;
-  evidence: SensitivityEvidence[];
-  nonSensitiveEvidence?: SensitivityEvidence;
-  observedValues: number;
-  nerCandidates: string[];
-  coverage: "metadata" | "complete" | "sampled" | "no_values";
-  sampledTarget: number;
-}
-
-/** Sole decision module for local column-level sensitivity assessments. */
-export class SensitivityClassifier {
-  constructor(
-    private readonly values: SensitivityValueSource,
-    private readonly detector?: LocalNerDetector,
-    private readonly options: {
-      queryTimeoutMs?: number;
-      nerConfidenceThreshold?: number;
-      maxNerValuesPerColumn?: number;
-      maxNerCandidatesPerTable?: number;
-      now?: () => number;
-    } = {},
-  ) {}
-
-  async assess(
-    targets: readonly SensitivityTableTarget[],
-    signal: AbortSignal,
-    sharedNerBudget?: SensitivityNerBudget,
-    onActivity?: (message: string) => void | Promise<void>,
-  ): Promise<readonly SensitivityColumnAssessment[]> {
-    const now = this.options.now ?? Date.now;
-    const maxNerValuesPerColumn = boundedCount(this.options.maxNerValuesPerColumn, 8, 8);
-    const states = new Map<string, ColumnState>();
-    for (const target of targets) {
-      for (const column of target.columns) {
-        const nonSensitiveEvidence = nonSensitiveStructuralEvidence(column);
-        const metadataMatch = nonSensitiveEvidence ? undefined : metadataEvidence(column);
-        const binary = !nonSensitiveEvidence && UNSUPPORTED_BINARY_TYPE.test(column.dataType);
-        states.set(column.id, {
-          column,
-          evidence: metadataMatch
-            ? [metadataMatch]
-            : binary
-              ? [{ kind: "type", ruleId: "type.binary_uninspectable" }]
-              : [],
-          ...(nonSensitiveEvidence ? { nonSensitiveEvidence } : {}),
-          observedValues: 0,
-          nerCandidates: [],
-          coverage: nonSensitiveEvidence || metadataMatch || binary ? "metadata" : "no_values",
-          sampledTarget: 0,
-        });
-      }
-    }
-
-    const completeTables = new Set<string>();
-    for (const [phaseIndex, phase] of SENSITIVITY_SAMPLE_PHASES.entries()) {
-      for (let offset = 0; offset < targets.length; offset += MAX_CONCURRENT_TABLE_SCANS) {
-        signal.throwIfAborted();
-        const batchNumber = Math.floor(offset / MAX_CONCURRENT_TABLE_SCANS) + 1;
-        const batchCount = Math.ceil(targets.length / MAX_CONCURRENT_TABLE_SCANS);
-        await onActivity?.(
-          `Scanning source data: pass ${phaseIndex + 1} of ${SENSITIVITY_SAMPLE_PHASES.length}, table batch ${batchNumber} of ${batchCount}.`,
-        );
-        signal.throwIfAborted();
-        const peerController = new AbortController();
-        const scanSignal = AbortSignal.any([signal, peerController.signal]);
-        try {
-          await Promise.all(targets.slice(offset, offset + MAX_CONCURRENT_TABLE_SCANS).map(async (target) => {
-            if (completeTables.has(target.table.id)) return;
-            const columns = target.columns.filter((column) => {
-              const state = states.get(column.id)!;
-              return state.evidence.length === 0 && !state.nonSensitiveEvidence
-                && (!phase.deepTextOnly || DEEP_TEXT_TYPE.test(column.dataType));
-            });
-            if (columns.length === 0) return;
-            const coverage = await this.values.scanTable({
-              ...target,
-              columns,
-              valuesPerColumn: phase.additionalValuesPerColumn,
-              sampleOffset: phase.targetValuesPerColumn - phase.additionalValuesPerColumn,
-              sampleSeed: phase.sampleSeed,
-              queryTimeoutMs: this.options.queryTimeoutMs ?? 5_000,
-              ...(phaseIndex === 0 ? { fullScanThreshold: 1_000 } : {}),
-            }, (batch) => {
-              for (const item of batch) {
-                if (item.value === null) continue;
-                const state = states.get(item.columnId);
-                if (!state || state.evidence.length > 0) continue;
-                state.observedValues += 1;
-                if ((item.characterLength ?? item.value.length) > 500) {
-                  state.evidence.push({ kind: "length", ruleId: "text.over_500_characters" });
-                  continue;
-                }
-                const match = contentEvidence(item.value);
-                if (match) {
-                  state.evidence.push(match);
-                  continue;
-                }
-                if (state.nerCandidates.length < maxNerValuesPerColumn
-                  && !state.nerCandidates.includes(item.value)) {
-                  state.nerCandidates.push(item.value);
-                }
-              }
-            }, scanSignal);
-            for (const column of columns) {
-              const state = states.get(column.id)!;
-              state.sampledTarget = Math.max(state.sampledTarget, phase.targetValuesPerColumn);
-              state.coverage = coverage.kind === "complete"
-                ? "complete"
-                : state.observedValues === 0 ? "no_values" : "sampled";
-            }
-            if (coverage.kind === "complete") completeTables.add(target.table.id);
-          }));
-        } catch (error) {
-          peerController.abort(error);
-          throw error;
-        }
-      }
-    }
-
-    const nerBudget = sharedNerBudget ?? { remainingMs: 10_000 };
-    if (this.detector && (this.detector.isReady?.() ?? true) && !signal.aborted
-      && nerBudget.remainingMs > 0) {
-      const maxCandidates = boundedCount(this.options.maxNerCandidatesPerTable, 2, 1_024);
-      const threshold = this.options.nerConfidenceThreshold ?? 0.8;
-      for (const [targetIndex, target] of targets.entries()) {
-        signal.throwIfAborted();
-        if (nerBudget.remainingMs <= 0) break;
-        const candidates: LocalNerCandidate[] = [];
-        candidateSelection: for (let valueIndex = 0; valueIndex < maxNerValuesPerColumn; valueIndex += 1) {
-          for (const column of target.columns) {
-            const state = states.get(column.id)!;
-            if (state.evidence.length > 0 || state.nonSensitiveEvidence) continue;
-            const text = state.nerCandidates[valueIndex];
-            if (text === undefined) continue;
-            candidates.push({ columnId: column.id, text });
-            if (candidates.length >= maxCandidates) break candidateSelection;
-          }
-        }
-        if (candidates.length === 0) continue;
-        await onActivity?.(
-          `Running local entity detection: table ${targetIndex + 1} of ${targets.length}.`,
-        );
-        signal.throwIfAborted();
-        const startedAt = now();
-        const deadline = startedAt + nerBudget.remainingMs;
-        try {
-          for (let offset = 0; offset < candidates.length; offset += MAX_NER_CANDIDATES_PER_REQUEST) {
-            if (signal.aborted || now() >= deadline) break;
-            try {
-              const detected = await this.detector.detect(
-                candidates.slice(offset, offset + MAX_NER_CANDIDATES_PER_REQUEST),
-                signal,
-                deadline,
-              );
-              for (const item of detected) {
-                const state = states.get(item.columnId);
-                if (!state || state.evidence.length > 0 || !Number.isFinite(item.confidence)
-                  || item.confidence < threshold || item.confidence > 1) continue;
-                const label = normalizedName(item.label).slice(0, 80);
-                if (!label) continue;
-                state.evidence.push({
-                  kind: "ner",
-                  ruleId: "ner.entity",
-                  label,
-                  confidence: item.confidence,
-                });
-              }
-            } catch {
-              // NER is optional: deterministic findings and scan coverage remain authoritative.
-              break;
-            }
-          }
-        } finally {
-          nerBudget.remainingMs = Math.max(0, nerBudget.remainingMs - Math.max(1, now() - startedAt));
-        }
-      }
-    }
-
-    return targets.flatMap((target) => target.columns.map((column) => {
-      const state = states.get(column.id)!;
-      const sensitive = state.evidence.length > 0;
-      const coverage = state.nonSensitiveEvidence
-        ? "metadata"
-        : state.observedValues === 0 && !sensitive ? "no_values" : state.coverage;
-      const coverageEvidence: SensitivityEvidence[] = sensitive
-        ? state.evidence
-        : state.nonSensitiveEvidence
-          ? [state.nonSensitiveEvidence]
-          : [{
-            kind: "coverage",
-            ruleId: coverage === "complete"
-              ? "coverage.complete"
-              : coverage === "no_values"
-                ? "coverage.no_values"
-                : `coverage.sampled_${state.sampledTarget}`,
-          }];
-      return {
-        columnId: column.id,
-        assessment: sensitive ? "sensitive" : "non_sensitive",
-        proposedSensitive: sensitive,
-        evidence: coverageEvidence,
-        observedValues: state.observedValues,
-        coverage,
-      };
-    }));
-  }
-
-  /** Convenience for focused callers and rule-level tests. Production orchestration uses assess(). */
-  async assessTable(
-    target: SensitivityTableTarget,
-    signal: AbortSignal,
-    _retiredRunDeadline?: number,
-    nerBudget?: SensitivityNerBudget,
-  ): Promise<readonly SensitivityColumnAssessment[]> {
-    return await this.assess([target], signal, nerBudget);
-  }
-}

backend/src/catalog/sensitivity-shadow.ts

@@ -1,100 +0,0 @@
-import { dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-import { loadConfig } from "../config.js";
-import { WorkspaceSecretStore } from "../workspaces/secret-store.js";
-import { PythonLocalNerDetector } from "./local-ner-detector.js";
-import { ConcreteCatalogPostgresAccess } from "./postgres-access.js";
-import { createCatalogRepository } from "./repository.js";
-import {
-  SENSITIVITY_POLICY_VERSION,
-  SensitivityAnalysisService,
-} from "./sensitivity-analysis-service.js";
-import { SensitivityClassifier } from "./sensitivity-classifier.js";
-import { ConcreteSensitivityValueSource } from "./sensitivity-value-source.js";
-
-const WORKSPACE_ID = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u;
-
-async function main(): Promise<void> {
-  const workspaceId = process.argv[2];
-  if (!workspaceId || !WORKSPACE_ID.test(workspaceId)) {
-    process.stderr.write("Usage: sensitivity-shadow <workspace-id>\n");
-    process.exitCode = 2;
-    return;
-  }
-
-  let detector: PythonLocalNerDetector | undefined;
-  let stage = "configuration";
-  try {
-    const config = loadConfig(process.env);
-    stage = "catalog";
-    const repository = createCatalogRepository(config.catalogDatabase);
-    if (!(await repository.available())) throw new Error("catalog unavailable");
-    const database = await repository.getByWorkspace(workspaceId);
-    if (!database) throw new Error("database unavailable");
-    stage = "source";
-    const secretStore = new WorkspaceSecretStore({
-      root: config.workspaceSecretStoreRoot,
-      runtimeRoot: config.workspaceSecretRuntimeRoot,
-      installationId: config.workspaceRegistry.installationId,
-    });
-    const access = new ConcreteCatalogPostgresAccess(secretStore, {
-      connectTimeoutMs: config.workspaceDiagnosticTimeoutMs,
-    });
-    const source = new ConcreteSensitivityValueSource(access, secretStore);
-    if (config.sensitivityNer) {
-      const workerScript = config.sensitivityNer.workerScript
-        ?? fileURLToPath(new URL("../../python/sensitivity_ner_worker.py", import.meta.url));
-      detector = new PythonLocalNerDetector({
-        pythonExecutable: config.sensitivityNer.pythonExecutable,
-        workerScript,
-        modelPath: config.sensitivityNer.modelPath,
-        cwd: dirname(workerScript),
-        threads: config.sensitivityNer.threads,
-      });
-      try {
-        await detector.warmup();
-      } catch {
-        await detector.close();
-        detector = undefined;
-      }
-    }
-    const startedAt = Date.now();
-    stage = "analysis";
-    const suggestions = await new SensitivityAnalysisService(
-      repository,
-      new SensitivityClassifier(source, detector),
-    ).analyze(database.id, "all", [], new AbortController().signal);
-    const assessments = { sensitive: 0, nonSensitive: 0 };
-    const coverage = { metadata: 0, complete: 0, sampled: 0, noValues: 0 };
-    const rules = new Map<string, number>();
-    for (const suggestion of suggestions) {
-      if (suggestion.assessment === "sensitive") assessments.sensitive += 1;
-      else assessments.nonSensitive += 1;
-      if (suggestion.coverage === "no_values") coverage.noValues += 1;
-      else coverage[suggestion.coverage] += 1;
-      for (const evidence of suggestion.evidence) {
-        rules.set(evidence.ruleId, (rules.get(evidence.ruleId) ?? 0) + 1);
-      }
-    }
-    process.stdout.write(`${JSON.stringify({
-      ok: true,
-      policyVersion: SENSITIVITY_POLICY_VERSION,
-      nerEnabled: detector !== undefined,
-      total: suggestions.length,
-      assessments,
-      coverage,
-      rules: Object.fromEntries([...rules].sort(([left], [right]) => left.localeCompare(right))),
-      elapsedMs: Date.now() - startedAt,
-    })}\n`);
-  } catch {
-    process.stdout.write(`${JSON.stringify({
-      ok: false,
-      code: `sensitivity_shadow_${stage}_failed`,
-    })}\n`);
-    process.exitCode = 1;
-  } finally {
-    await detector?.close();
-  }
-}
-
-await main();

backend/src/catalog/sensitivity-value-source.ts

@@ -1,291 +0,0 @@
-import { readFile } from "node:fs/promises";
-import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
-import { CATALOG_SECRET_IDS } from "./secrets.js";
-import type { CatalogPostgresAccess } from "./postgres-access.js";
-import type {
-  SensitivityScanCoverage,
-  SensitivityScanRequest,
-  SensitivityValueObservation,
-  SensitivityValueSource,
-} from "./sensitivity-classifier.js";
-import { CatalogConnectorError, type CatalogColumn } from "./types.js";
-
-const MAX_VALUE_CHARACTERS = 501;
-const MAX_COLUMNS_PER_QUERY = 25;
-const SAMPLE_OVERSCAN_FACTOR = 10;
-
-function quoteIdentifier(identifier: string): string {
-  return `"${identifier.replaceAll('"', '""')}"`;
-}
-
-function chunks<T>(items: readonly T[], size: number): T[][] {
-  const result: T[][] = [];
-  for (let offset = 0; offset < items.length; offset += size) {
-    result.push(items.slice(offset, offset + size));
-  }
-  return result;
-}
-
-function tableReference(request: SensitivityScanRequest): string {
-  return `${quoteIdentifier(request.database.schema)}.${quoteIdentifier(request.table.name)}`;
-}
-
-function samplePercentage(valuesPerColumn: number): number {
-  if (valuesPerColumn <= 300) return 30;
-  if (valuesPerColumn <= 700) return 70;
-  return 100;
-}
-
-function flatValueQuery(
-  request: SensitivityScanRequest,
-  columns: readonly CatalogColumn[],
-  options: { complete: boolean; randomized: boolean },
-): string {
-  const projections = columns.map((column) => quoteIdentifier(column.name)).join(", ");
-  const perColumnLimit = options.complete
-    ? request.fullScanThreshold ?? request.valuesPerColumn
-    : request.valuesPerColumn;
-  const rowLimit = Math.max(perColumnLimit, perColumnLimit * SAMPLE_OVERSCAN_FACTOR);
-  const sample = options.complete
-    ? `SELECT ${projections} FROM ${tableReference(request)}`
-    : [
-      `SELECT ${projections} FROM ${tableReference(request)}`,
-      ...(options.randomized
-        ? [`TABLESAMPLE SYSTEM (${samplePercentage(request.valuesPerColumn)}) REPEATABLE (${request.sampleSeed})`]
-        : []),
-      `LIMIT ${rowLimit} OFFSET ${request.sampleOffset}`,
-    ].join(" ");
-  const values = columns.map((column, index) => {
-    const identifier = quoteIdentifier(column.name);
-    return [
-      `(${index}, LEFT((sampled.${identifier})::text, ${MAX_VALUE_CHARACTERS}),`,
-      `CASE WHEN sampled.${identifier} IS NULL THEN NULL`,
-      `ELSE char_length((sampled.${identifier})::text) END)`,
-    ].join(" ");
-  }).join(", ");
-  return [
-    `WITH sampled AS MATERIALIZED (${sample}),`,
-    "ranked AS (",
-    "SELECT value.__column_index, value.__value, value.__length,",
-    "row_number() OVER (PARTITION BY value.__column_index) AS __rank",
-    "FROM sampled",
-    `CROSS JOIN LATERAL (VALUES ${values}) AS value(__column_index, __value, __length)`,
-    "WHERE value.__value IS NOT NULL",
-    ")",
-    "SELECT __column_index, __value, __length FROM ranked",
-    `WHERE __rank <= ${perColumnLimit}`,
-  ].join(" ");
-}
-
-function observations(
-  columns: readonly CatalogColumn[],
-  rows: readonly Record<string, unknown>[],
-): SensitivityValueObservation[] {
-  return rows.flatMap((row) => {
-    const index = Number(row.__column_index);
-    const column = Number.isSafeInteger(index) && index >= 0 ? columns[index] : undefined;
-    if (!column || row.__value === null || row.__value === undefined) return [];
-    const value = String(row.__value);
-    const parsedLength = row.__length === null || row.__length === undefined
-      ? null
-      : Number(row.__length);
-    return [{
-      columnId: column.id,
-      value,
-      characterLength: parsedLength !== null && Number.isSafeInteger(parsedLength) && parsedLength >= 0
-        ? parsedLength
-        : value.length,
-    }];
-  });
-}
-
-function cancelled(error: unknown): boolean {
-  return Boolean(error && typeof error === "object" && "code" in error && error.code === "57014");
-}
-
-/**
- * Database-specific sampling adapter. Policy stays in SensitivityClassifier; this module only
- * produces bounded, normalized non-null observations without persisting or logging values.
- */
-export class ConcreteSensitivityValueSource implements SensitivityValueSource {
-  constructor(
-    private readonly access: CatalogPostgresAccess,
-    private readonly secretStore?: Pick<WorkspaceSecretStore, "materialize">,
-  ) {}
-
-  async scanTable(
-    request: SensitivityScanRequest,
-    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
-    signal: AbortSignal,
-  ): Promise<SensitivityScanCoverage> {
-    if (request.columns.length === 0) return { kind: "complete", observedValues: 0 };
-    if (request.database.binding.transport === "rest_api") {
-      return await this.scanRest(request, consume, signal);
-    }
-    return await this.scanPostgres(request, consume, signal);
-  }
-
-  private async scanPostgres(
-    request: SensitivityScanRequest,
-    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
-    signal: AbortSignal,
-  ): Promise<SensitivityScanCoverage> {
-    const client = await this.access.connect(request.database, signal);
-    let transactionOpen = false;
-    let savepointSequence = 0;
-    let observedValues = 0;
-    try {
-      signal.throwIfAborted();
-      await client.query("BEGIN TRANSACTION READ ONLY", []);
-      transactionOpen = true;
-      await client.query("SELECT set_config('statement_timeout', $1, true)", [
-        `${Math.max(1, Math.floor(request.queryTimeoutMs))}ms`,
-      ]);
-      const boundedQuery = async (sql: string): Promise<Array<Record<string, unknown>> | undefined> => {
-        signal.throwIfAborted();
-        savepointSequence += 1;
-        const savepoint = `sensitivity_scan_${savepointSequence}`;
-        await client.query(`SAVEPOINT ${savepoint}`, []);
-        try {
-          return (await client.query(sql, [])).rows;
-        } catch (error) {
-          if (!cancelled(error)) throw error;
-          await client.query(`ROLLBACK TO SAVEPOINT ${savepoint}`, []);
-          return undefined;
-        } finally {
-          await client.query(`RELEASE SAVEPOINT ${savepoint}`, []).catch(() => undefined);
-        }
-      };
-
-      let complete = false;
-      if (request.fullScanThreshold !== undefined) {
-        const probe = await boundedQuery(
-          `SELECT 1 AS __present FROM ${tableReference(request)} LIMIT ${request.fullScanThreshold + 1}`,
-        );
-        complete = probe !== undefined && probe.length <= request.fullScanThreshold;
-      }
-      for (const columnChunk of chunks(request.columns, MAX_COLUMNS_PER_QUERY)) {
-        signal.throwIfAborted();
-        let rows = await boundedQuery(flatValueQuery(request, columnChunk, {
-          complete,
-          randomized: !complete,
-        }));
-        if (rows === undefined && complete) {
-          complete = false;
-          rows = await boundedQuery(flatValueQuery(request, columnChunk, {
-            complete: false,
-            randomized: true,
-          }));
-        }
-        if (!complete && (rows === undefined || rows.length === 0)) {
-          rows = await boundedQuery(flatValueQuery(request, columnChunk, {
-            complete: false,
-            randomized: false,
-          }));
-        }
-        if (rows === undefined) throw new CatalogConnectorError("Sensitivity sample query timed out");
-        const batch = observations(columnChunk, rows);
-        observedValues += batch.length;
-        if (batch.length > 0) await consume(batch);
-      }
-      return { kind: complete ? "complete" : "sampled", observedValues };
-    } catch (error) {
-      if (error instanceof CatalogConnectorError) throw error;
-      throw new CatalogConnectorError("Sensitivity source scan failed");
-    } finally {
-      if (transactionOpen) await client.query("ROLLBACK", []).catch(() => undefined);
-      await client.end().catch(() => undefined);
-    }
-  }
-
-  private async scanRest(
-    request: SensitivityScanRequest,
-    consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
-    signal: AbortSignal,
-  ): Promise<SensitivityScanCoverage> {
-    if (!this.secretStore) throw new CatalogConnectorError("REST sensitivity scanning is not configured");
-    const auth = request.database.binding.restAuth ?? "bearer";
-    const materialized = this.secretStore.materialize(
-      request.database.workspaceId,
-      auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
-    );
-    let observedValues = 0;
-    try {
-      const headers: Record<string, string> = { "content-type": "application/json" };
-      if (auth !== "none") {
-        const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
-        if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
-        const credential = (await readFile(credentialFile, "utf8")).trim();
-        if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
-        else headers["x-api-key"] = credential;
-      }
-      const baseUrl = request.database.binding.baseUrl?.replace(/\/+$/u, "");
-      if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete");
-      const runQuery = async (sql: string): Promise<Array<Record<string, unknown>> | undefined> => {
-        const timeout = AbortSignal.timeout(Math.max(1, Math.floor(request.queryTimeoutMs)));
-        try {
-          const response = await fetch(`${baseUrl}/rpc/run_query`, {
-            method: "POST",
-            headers,
-            body: JSON.stringify({ query_text: sql }),
-            signal: AbortSignal.any([signal, timeout]),
-          });
-          if (!response.ok) throw new CatalogConnectorError("REST sensitivity source scan failed");
-          const body: unknown = await response.json();
-          if (!Array.isArray(body)
-            || body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
-            throw new CatalogConnectorError("REST sensitivity source response is invalid");
-          }
-          return body as Array<Record<string, unknown>>;
-        } catch (error) {
-          if (signal.aborted) throw error;
-          if (timeout.aborted) return undefined;
-          throw error;
-        }
-      };
-
-      let complete = false;
-      if (request.fullScanThreshold !== undefined) {
-        const probe = await runQuery(
-          `SELECT 1 AS __present FROM ${tableReference(request)} LIMIT ${request.fullScanThreshold + 1}`,
-        );
-        complete = probe !== undefined && probe.length <= request.fullScanThreshold;
-      }
-      let requestCount = request.fullScanThreshold === undefined ? 0 : 1;
-      for (const columnChunk of chunks(request.columns, MAX_COLUMNS_PER_QUERY)) {
-        signal.throwIfAborted();
-        let rows = await runQuery(flatValueQuery(request, columnChunk, {
-          complete,
-          randomized: !complete,
-        }));
-        requestCount += 1;
-        if (rows === undefined && complete) {
-          complete = false;
-          rows = await runQuery(flatValueQuery(request, columnChunk, {
-            complete: false,
-            randomized: true,
-          }));
-          requestCount += 1;
-        }
-        if (!complete && (rows === undefined || rows.length === 0)) {
-          rows = await runQuery(flatValueQuery(request, columnChunk, {
-            complete: false,
-            randomized: false,
-          }));
-          requestCount += 1;
-        }
-        if (rows === undefined) throw new CatalogConnectorError("REST sensitivity sample query timed out");
-        const batch = observations(columnChunk, rows);
-        observedValues += batch.length;
-        if (batch.length > 0) await consume(batch);
-      }
-      // Multiple HTTP requests cannot share a source snapshot, so only one-request reads are complete.
-      return { kind: complete && requestCount === 1 ? "complete" : "sampled", observedValues };
-    } catch (error) {
-      if (error instanceof CatalogConnectorError) throw error;
-      throw new CatalogConnectorError("REST sensitivity source scan failed");
-    } finally {
-      materialized.release();
-    }
-  }
-}

backend/src/catalog/service.ts

@@ -1,3 +1,5 @@
+import { buildInstallationContract } from "../workspaces/contracts.js";
+import { resolveBinding } from "../workspaces/bindings.js";
 import type { WorkspaceRegistry } from "../workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../workspaces/schema.js";
 import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
@@ -43,33 +45,60 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
   secrets: Record<CatalogSecretName, boolean>;
 }
 
+function bindingValue(workspace: WorkspaceDescriptor, values: Record<string, string>, suffix: string) {
+  const variable = buildInstallationContract(workspace).variables.find((entry) => (
+    entry.role === "DWH" && entry.suffix === suffix
+  ));
+  return variable ? values[variable.name] : undefined;
+}
+
+function numeric(value: string | undefined): number | undefined {
+  if (!value) return undefined;
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65_535 ? parsed : undefined;
+}
+
+function yamlBinding(workspace: WorkspaceDescriptor, secretRoots: readonly string[]): DatabaseBinding {
+  const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
+  const value = (suffix: string) => bindingValue(workspace, effective.values, suffix);
+  return {
+    transport: effective.transport,
+    host: value("HOST"),
+    port: numeric(value("PORT")) ?? workspace.dwh.port,
+    username: value("USER"),
+    baseUrl: value("BASE_URL"),
+    restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
+    restAuth: workspace.diagnostics?.dwh_rest?.auth ?? "bearer",
+    tlsServername: value("TLS_SERVERNAME"),
+    sshHost: value("SSH_HOST"),
+    sshPort: numeric(value("SSH_PORT")),
+    sshUsername: value("SSH_USER"),
+    sshTargetHost: value("SSH_TARGET_HOST"),
+    sshTargetPort: numeric(value("SSH_TARGET_PORT")),
+  };
+}
+
 function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<CatalogSecretName, boolean> {
   return Object.fromEntries(Object.entries(CATALOG_SECRET_IDS).map(([name, id]) => (
     [name, store.has(workspaceId, id)]
   ))) as Record<CatalogSecretName, boolean>;
 }
 
-function databaseRuntimeState(
+function workspaceRuntimeState(
   store: WorkspaceSecretStore,
-  database: WorkspaceDatabase,
+  workspace: WorkspaceDescriptor,
+  secretRoots: readonly string[],
 ): NonNullable<CatalogListItem["runtimeBinding"]> {
-  const { binding, workspaceId } = database;
-  const configured = (id: string) => store.has(workspaceId, id);
-  const connectionComplete = binding.transport === "postgres_direct"
-    ? Boolean(binding.host && binding.port && binding.username && configured(CATALOG_SECRET_IDS.password))
-    : binding.transport === "rest_api"
-      ? Boolean(binding.baseUrl && (binding.restAuth === "none" || configured(CATALOG_SECRET_IDS.apiKey)))
-      : Boolean(
-        binding.username && binding.sshHost && binding.sshPort && binding.sshUsername
-        && binding.sshTargetHost && binding.sshTargetPort
-        && configured(CATALOG_SECRET_IDS.password)
-        && configured(CATALOG_SECRET_IDS.sshPrivateKey)
-        && configured(CATALOG_SECRET_IDS.sshKnownHosts),
-      );
+  const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
+  const secretVariables = new Set(requirements.map(({ variable }) => variable));
+  const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
+  const configurationRequired = requirements.some(({ id, required }) => (
+    required && !store.has(workspace.workspace.id, id)
+  )) || effective.missing.some((variable) => !secretVariables.has(variable));
   return {
-    transport: binding.transport,
-    configurationState: connectionComplete ? "ready" : "configuration_required",
-    sessionTransportSupported: binding.transport !== "ssh_tunnel",
+    transport: effective.transport,
+    configurationState: configurationRequired ? "configuration_required" : "ready",
+    sessionTransportSupported: effective.transport !== "ssh_tunnel",
   };
 }
 
@@ -116,18 +145,17 @@ export class CatalogService {
     const active = await Promise.all(workspaces.map(async (entry) => {
       const database = byWorkspace.get(entry.id);
       const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
+      const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
       const base = database ?? {
         workspaceId: entry.id,
         engine: "postgres" as const,
-        databaseName: "",
-        schema: "",
+        databaseName: workspace.dwh.database,
+        schema: workspace.dwh.schema,
         version: 0,
         createdAt: "",
         updatedAt: "",
-        binding: { transport: "postgres_direct" as const },
+        binding: runtimeDatabaseBinding,
         connectionStatus: "untested" as const,
-        metadataContentRevision: 0,
-        preprocessingStatus: "failed" as const,
       };
       return {
         ...base,
@@ -139,7 +167,7 @@ export class CatalogService {
           blob: entry.revision.blob,
         },
         workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
-        runtimeBinding: database ? databaseRuntimeState(this.secretStore, database) : null,
+        runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
         configured: database !== undefined,
         secrets: secretState(this.secretStore, entry.id),
       };
@@ -167,13 +195,13 @@ export class CatalogService {
   }
 
   async normalizeInput(input: DatabaseConfigurationInput): Promise<DatabaseConfigurationInput> {
-    await this.ensureWorkspace(input.workspaceId);
+    const workspace = await this.ensureWorkspace(input.workspaceId);
     if (input.binding.transport !== "rest_api") return input;
     return {
       ...input,
       binding: {
         ...input.binding,
-        restPath: input.binding.restPath ?? "/health",
+        restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
       },
     };
   }

backend/src/catalog/sync-worker.ts

@@ -39,10 +39,7 @@ function safeFailure(error: unknown): { code: string; message: string } {
     };
   }
   if (error instanceof CatalogConnectorError) {
-    return {
-      code: "schema_introspection_failed",
-      message: "The database schema could not be read. Check the connection and credentials, then try again.",
-    };
+    return { code: "schema_introspection_failed", message: "The database schema could not be read safely." };
   }
   return { code: "schema_sync_failed", message: "Schema synchronization failed." };
 }
@@ -67,6 +64,7 @@ export class CatalogSyncWorker {
   }
 
   async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise<CatalogSyncRun> {
+    this.assertReady(database);
     const uniqueTableIds = [...new Set(tableIds)];
     if (scope === "columns") {
       const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId)));
@@ -179,6 +177,7 @@ export class CatalogSyncWorker {
       if (!database || database.version !== claimed.requestedDatabaseVersion) {
         throw new CatalogConflictError("Database binding changed before synchronization started");
       }
+      this.assertReady(database);
       const progress: CatalogSchemaScanProgress = async (phase, counts) => {
         await this.checkCancelled(runId);
         await this.repository.updateSyncRun(runId, {
@@ -278,6 +277,12 @@ export class CatalogSyncWorker {
     }
   }
 
+  private assertReady(database: WorkspaceDatabase): void {
+    if (database.connectionStatus !== "reachable" || database.testedVersion !== database.version) {
+      throw new CatalogConflictError("Test the current database binding before synchronizing its schema");
+    }
+  }
+
   private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void {
     const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const;
     for (const name of required) {

backend/src/catalog/types.ts

@@ -2,7 +2,6 @@ export const DATABASE_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"]
 export type DatabaseTransport = (typeof DATABASE_TRANSPORTS)[number];
 
 export type ConnectionStatus = "untested" | "reachable" | "failed";
-export type CatalogPreprocessingStatus = "running" | "succeeded" | "failed";
 
 export interface DatabaseBinding {
   transport: DatabaseTransport;
@@ -37,23 +36,8 @@ export interface WorkspaceDatabase {
   lastErrorMessage?: string;
   schemaSyncedVersion?: number;
   schemaSyncedAt?: string;
-  metadataContentRevision: number;
-  preprocessingStatus: CatalogPreprocessingStatus;
-  preprocessingInputFingerprint?: string;
-  preprocessedMetadataRevision?: number;
-  preprocessingStartedAt?: string;
-  preprocessingFinishedAt?: string;
-  preprocessingErrorCode?: string;
 }
 
-export type CatalogPreprocessingStartResult =
-  | { kind: "started"; database: WorkspaceDatabase }
-  | { kind: "not_found" | "schema_stale" | "catalog_busy" | "already_running" };
-
-export type CatalogPreprocessingClearResult =
-  | { kind: "cleared"; database: WorkspaceDatabase }
-  | { kind: "not_found" | "already_running" };
-
 export interface CatalogMetrics {
   scope: "global" | "database";
   databaseId: string | null;
@@ -118,7 +102,6 @@ export interface CatalogColumn {
   description: string | null;
   generatedDescription: string | null;
   sensitive: boolean;
-  sensitivityReason: string | null;
   lastSyncedDatabaseVersion: number | null;
   lastSyncedAt: string | null;
   version: number;
@@ -212,7 +195,7 @@ export interface CatalogLogicalRelationshipCandidate {
 
 export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
 export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
-export type CatalogDescriptionTarget = "tables" | "columns" | "database" | "database_columns";
+export type CatalogDescriptionTarget = "tables" | "columns";
 
 export interface CatalogMetadataDeleteCounts {
   tables: number;
@@ -283,21 +266,18 @@ export interface DescriptionGenerationEvent {
   createdAt: string;
 }
 
-export type SensitivityAnalysisScope = "all" | "selected_tables" | "selected_columns";
-export type SensitivityAnalysisStatus = "running" | "completed" | "failed" | "interrupted";
+export type SensitiveDataSuggestionScope = "all" | "selected_tables" | "selected_columns";
+export type SensitiveDataSuggestionStatus = "running" | "completed" | "failed" | "interrupted";
 
-export interface SensitivityAnalysisRun {
+export interface SensitiveDataSuggestionRun {
   id: string;
   databaseId: string;
-  scope: SensitivityAnalysisScope;
-  engine: "llm" | "local";
-  modelId: string | null;
-  policyVersion: string | null;
-  status: SensitivityAnalysisStatus;
+  scope: SensitiveDataSuggestionScope;
+  modelId: string;
+  status: SensitiveDataSuggestionStatus;
   total: number;
   suggestedSensitive: number;
   suggestedNonSensitive: number;
-  unknown: number;
   inputTokens: number;
   cacheReadTokens: number;
   outputTokens: number;
@@ -308,12 +288,11 @@ export interface SensitivityAnalysisRun {
   errorSummary: string | null;
 }
 
-export interface SensitivityAnalysisRunUpdate {
-  status?: SensitivityAnalysisStatus;
+export interface SensitiveDataSuggestionRunUpdate {
+  status?: SensitiveDataSuggestionStatus;
   total?: number;
   suggestedSensitive?: number;
   suggestedNonSensitive?: number;
-  unknown?: number;
   finishedAt?: string | null;
   errorSummary?: string | null;
   inputTokens?: number;
@@ -321,7 +300,7 @@ export interface SensitivityAnalysisRunUpdate {
   outputTokens?: number;
 }
 
-export interface SensitivityAnalysisEvent {
+export interface SensitiveDataSuggestionEvent {
   runId: string;
   sequence: number;
   level: "info" | "warning" | "error";
@@ -449,17 +428,6 @@ export interface CatalogRepository {
   list(): Promise<WorkspaceDatabase[]>;
   get(id: string): Promise<WorkspaceDatabase | undefined>;
   getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined>;
-  beginPreprocessing(
-    workspaceId: string,
-    inputFingerprint: string,
-  ): Promise<CatalogPreprocessingStartResult>;
-  finishPreprocessing(
-    workspaceId: string,
-    metadataContentRevision: number,
-    inputFingerprint: string,
-    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
-  ): Promise<WorkspaceDatabase | undefined>;
-  clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult>;
   getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined>;
   create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase>;
   update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined>;
@@ -491,7 +459,6 @@ export interface CatalogRepository {
     description: string | null,
     generatedDescription: string | null,
     sensitive?: boolean,
-    sensitivityReason?: string | null,
   ): Promise<CatalogColumn | undefined>;
   consolidateGeneratedDescriptions(
     databaseId: string,
@@ -524,29 +491,29 @@ export interface CatalogRepository {
     runId: string,
     afterSequence?: number,
   ): Promise<DescriptionGenerationEvent[]>;
-  createSensitivityAnalysisRun(
+  createSensitiveDataSuggestionRun(
     databaseId: string,
-    scope: SensitivityAnalysisScope,
-    origin: { engine: "llm"; modelId: string } | { engine: "local"; policyVersion: string },
-  ): Promise<SensitivityAnalysisRun>;
-  getSensitivityAnalysisRun(runId: string): Promise<SensitivityAnalysisRun | undefined>;
-  listSensitivityAnalysisRuns(limit?: number): Promise<SensitivityAnalysisRun[]>;
-  interruptActiveSensitivityAnalysisRuns(
+    scope: SensitiveDataSuggestionScope,
+    modelId: string,
+  ): Promise<SensitiveDataSuggestionRun>;
+  getSensitiveDataSuggestionRun(runId: string): Promise<SensitiveDataSuggestionRun | undefined>;
+  listSensitiveDataSuggestionRuns(limit?: number): Promise<SensitiveDataSuggestionRun[]>;
+  interruptActiveSensitiveDataSuggestionRuns(
     errorSummary: string,
-  ): Promise<SensitivityAnalysisRun[]>;
-  updateSensitivityAnalysisRun(
+  ): Promise<SensitiveDataSuggestionRun[]>;
+  updateSensitiveDataSuggestionRun(
     runId: string,
-    update: SensitivityAnalysisRunUpdate,
-  ): Promise<SensitivityAnalysisRun | undefined>;
-  appendSensitivityAnalysisEvent(
+    update: SensitiveDataSuggestionRunUpdate,
+  ): Promise<SensitiveDataSuggestionRun | undefined>;
+  appendSensitiveDataSuggestionEvent(
     runId: string,
-    level: SensitivityAnalysisEvent["level"],
+    level: SensitiveDataSuggestionEvent["level"],
     message: string,
-  ): Promise<SensitivityAnalysisEvent>;
-  listSensitivityAnalysisEvents(
+  ): Promise<SensitiveDataSuggestionEvent>;
+  listSensitiveDataSuggestionEvents(
     runId: string,
     afterSequence?: number,
-  ): Promise<SensitivityAnalysisEvent[]>;
+  ): Promise<SensitiveDataSuggestionEvent[]>;
   listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]>;
   listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]>;
   getLogicalRelationshipContext(databaseId: string): Promise<CatalogLogicalRelationshipContext | undefined>;

backend/src/config.ts

@@ -32,13 +32,6 @@ export interface AppConfig {
   piManagementTimeoutMs: number;
   secretsFile?: string;
   installationConfigFile?: string;
-  modelCatalogFile?: string;
-  sensitivityNer?: {
-    pythonExecutable: string;
-    modelPath: string;
-    workerScript?: string;
-    threads: number;
-  };
   piAuthFile?: string;
   secretFiles: Readonly<Record<string, string | undefined>>;
   modelApiKeyFile?: string;
@@ -57,7 +50,6 @@ export interface AppConfig {
   workspaceSecretRuntimeRoot: string;
   internalQdrantUrl: string;
   internalEmbeddingUrl: string;
-  internalEmbeddingId: string;
   internalEmbeddingModel: string;
   internalEmbeddingDimensions: number;
 }
@@ -197,21 +189,6 @@ function positiveDimension(value: string | undefined, fallback: number): number
   return parsed;
 }
 
-function internalEmbeddingIdentity(
-  identityValue: string | undefined,
-  modelValue: string | undefined,
-): { id: string; model: string } {
-  const id = identityValue ?? "ollama/qwen3-embedding:0.6b";
-  if (!/^ollama\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/.test(id)) {
-    throw new Error("internal embedding identity configuration is invalid");
-  }
-  const model = id.slice(id.indexOf("/") + 1);
-  if (modelValue !== undefined && modelValue !== model) {
-    throw new Error("internal embedding model does not match its canonical identity");
-  }
-  return { id, model };
-}
-
 function catalogDatabase(env: Record<string, string | undefined>): CatalogConnectionConfig | undefined {
   const value = env.THT_CATALOG_DATABASE_URL;
   if (value !== undefined) {
@@ -353,42 +330,6 @@ export function loadConfig(
     || installationConfigFile.includes("\0")
     || !path.isAbsolute(installationConfigFile)
   )) throw new Error("installation configuration is invalid");
-  const modelCatalogFile = env.THT_MODEL_CATALOG_FILE;
-  if (modelCatalogFile !== undefined && (
-    modelCatalogFile.trim() !== modelCatalogFile
-    || modelCatalogFile.length === 0
-    || modelCatalogFile.includes("\0")
-    || !path.isAbsolute(modelCatalogFile)
-  )) throw new Error("runtime model catalog configuration is invalid");
-  const sensitivityNerModelPath = env.THT_SENSITIVITY_NER_MODEL_PATH;
-  const sensitivityNerPython = env.THT_SENSITIVITY_NER_PYTHON;
-  const sensitivityNerWorker = env.THT_SENSITIVITY_NER_WORKER;
-  for (const [value, label] of [
-    [sensitivityNerModelPath, "model path"],
-    [sensitivityNerPython, "Python executable"],
-    [sensitivityNerWorker, "worker path"],
-  ] as const) {
-    if (value !== undefined && (
-      value.length === 0 || value.trim() !== value || value.includes("\0") || !path.isAbsolute(value)
-    )) throw new Error(`sensitivity NER ${label} configuration is invalid`);
-  }
-  if (sensitivityNerModelPath === undefined && (
-    sensitivityNerPython !== undefined
-    || sensitivityNerWorker !== undefined
-    || env.THT_SENSITIVITY_NER_THREADS !== undefined
-  )) throw new Error("sensitivity NER settings require a model path");
-  const sensitivityNerThreads = Number(env.THT_SENSITIVITY_NER_THREADS ?? 2);
-  if (!Number.isSafeInteger(sensitivityNerThreads) || sensitivityNerThreads < 1 || sensitivityNerThreads > 8) {
-    throw new Error("sensitivity NER thread configuration is invalid");
-  }
-  const sensitivityNer = sensitivityNerModelPath === undefined
-    ? undefined
-    : {
-      modelPath: sensitivityNerModelPath,
-      pythonExecutable: sensitivityNerPython ?? "/opt/sensitivity-ner/bin/python",
-      ...(sensitivityNerWorker ? { workerScript: sensitivityNerWorker } : {}),
-      threads: sensitivityNerThreads,
-    };
   const piAuthFile = env.THT_PI_AUTH_FILE;
   if (piAuthFile !== undefined && (
     piAuthFile.trim() !== piAuthFile || piAuthFile.length === 0 || piAuthFile.includes("\0")
@@ -450,10 +391,6 @@ export function loadConfig(
     "internal embedding URL",
     ["embedding", "localhost"],
   );
-  const internalEmbedding = internalEmbeddingIdentity(
-    env.THT_INTERNAL_EMBEDDING_ID,
-    env.THT_INTERNAL_EMBEDDING_MODEL,
-  );
   return {
     host: env.HOST ?? "127.0.0.1",
     port: Number(env.PORT ?? 8787),
@@ -467,7 +404,7 @@ export function loadConfig(
     publicExposure,
     sessionStorage,
     catalogDatabase: catalogDatabase(env),
-    defaults: { thinking: env.PI_THINKING },
+    defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING },
     maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4),
     settingsFile,
     maintenanceFile: env.THT_MAINTENANCE_FILE ?? path.join(path.dirname(settingsFile), "maintenance.json"),
@@ -476,8 +413,6 @@ export function loadConfig(
     piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
     secretsFile,
     installationConfigFile,
-    modelCatalogFile,
-    sensitivityNer,
     piAuthFile,
     secretFiles,
     modelApiKeyFile,
@@ -490,8 +425,7 @@ export function loadConfig(
     workspaceSecretRuntimeRoot,
     internalQdrantUrl,
     internalEmbeddingUrl,
-    internalEmbeddingId: internalEmbedding.id,
-    internalEmbeddingModel: internalEmbedding.model,
+    internalEmbeddingModel: env.THT_INTERNAL_EMBEDDING_MODEL ?? "qwen3-embedding:0.6b",
     internalEmbeddingDimensions: positiveDimension(env.THT_INTERNAL_EMBEDDING_DIMENSIONS, 1024),
   };
 }

backend/src/config/secret-bundle.ts

@@ -8,7 +8,7 @@ import {
   isUsableAuthenticationSecret,
 } from "../auth/secret-policy.js";
 
-/** Credential names that Installation Model Catalog providers may reference. */
+/** Credential names that metadata-generation model entries may reference. */
 export const METADATA_GENERATION_SECRET_KEYS = Object.freeze([
   "THT_METADATA_API_KEY", "ANTHROPIC_API_KEY", "AZURE_API_KEY", "GEMINI_API_KEY",
   "DEEPSEEK_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ZAI_API_KEY",

backend/src/models/runtime-model-catalog.ts

@@ -1,161 +0,0 @@
-import {
-  closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, type Stats,
-} from "node:fs";
-import { z } from "zod";
-
-const MAX_CATALOG_BYTES = 1024 * 1024;
-const RUNTIME_CATALOG_FILE = "/run/thothii-model-catalog/catalog.json";
-const canonicalId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/);
-const secretBundleKey = /^[A-Z][A-Z0-9_]{0,63}$/;
-
-const endpointSchema = z.object({
-  baseUrl: z.string().url(),
-  apiVersion: z.string().optional(),
-}).strict();
-
-const authenticationSchema = z.object({
-  mode: z.enum(["secret_env", "pi_auth", "none"]),
-  apiKeyEnv: z.string().optional(),
-}).strict();
-
-const runtimeModelSchema = z.object({
-  id: canonicalId,
-  provider: z.string().min(1),
-  model: z.string().min(1),
-  label: z.string().min(1),
-  upstreamModel: z.string().min(1),
-  endpoint: endpointSchema.optional(),
-  authentication: authenticationSchema,
-  sessionAdapter: z.object({ mode: z.enum(["pi_builtin", "openai_compatible"]) }).strict().optional(),
-  metadataAdapter: z.object({ litellmProvider: z.string().min(1) }).strict().optional(),
-  session: z.object({
-    reasoning: z.boolean(),
-    input: z.array(z.string()).optional(),
-    cost: z.object({
-      input: z.number(), output: z.number(), cacheRead: z.number(), cacheWrite: z.number(),
-    }).strict().optional(),
-    contextWindow: z.number().int().positive().optional(),
-    maxTokens: z.number().int().positive().optional(),
-    compatibility: z.object({
-      supportsDeveloperRole: z.boolean(),
-      supportsReasoningEffort: z.boolean(),
-      supportsStore: z.boolean(),
-      maxTokensField: z.string().optional(),
-    }).strict().optional(),
-  }).strict().optional(),
-  metadataGeneration: z.object({ disableThinking: z.boolean() }).strict().optional(),
-}).strict();
-
-const catalogSchema = z.object({
-  schemaVersion: z.literal(1),
-  defaultSession: canonicalId,
-  defaultMetadataGeneration: canonicalId.optional(),
-  embedding: z.object({ id: canonicalId, dimensions: z.number().int().positive() }).strict(),
-  models: z.array(runtimeModelSchema).max(64),
-}).strict();
-
-export type RuntimeModel = z.infer<typeof runtimeModelSchema>;
-
-export interface RuntimeModelCatalog {
-  readonly defaultSession: string | null;
-  readonly defaultMetadataGeneration: string | null;
-  readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
-  sessionModels(): readonly RuntimeModel[];
-  metadataModels(): readonly RuntimeModel[];
-  hasSession(id: string): boolean;
-}
-
-class RestartLoadedRuntimeModelCatalog implements RuntimeModelCatalog {
-  readonly defaultSession: string | null;
-  readonly defaultMetadataGeneration: string | null;
-  readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
-  readonly #sessions: readonly RuntimeModel[];
-  readonly #metadata: readonly RuntimeModel[];
-  readonly #sessionIds: ReadonlySet<string>;
-
-  constructor(catalog?: z.infer<typeof catalogSchema>) {
-    this.defaultSession = catalog?.defaultSession ?? null;
-    this.defaultMetadataGeneration = catalog?.defaultMetadataGeneration ?? null;
-    this.embedding = catalog ? Object.freeze({ ...catalog.embedding }) : null;
-    this.#sessions = Object.freeze((catalog?.models ?? []).filter((model) => model.session !== undefined));
-    this.#metadata = Object.freeze((catalog?.models ?? []).filter((model) => model.metadataGeneration !== undefined));
-    this.#sessionIds = new Set(this.#sessions.map((model) => model.id));
-  }
-
-  sessionModels(): readonly RuntimeModel[] { return this.#sessions.map((model) => ({ ...model })); }
-  metadataModels(): readonly RuntimeModel[] { return this.#metadata.map((model) => ({ ...model })); }
-  hasSession(id: string): boolean { return this.#sessionIds.has(id); }
-}
-
-function protectedCatalogStat(file: string, info: Stats): boolean {
-  const mode = info.mode & 0o777;
-  if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
-    || info.size < 1 || info.size > MAX_CATALOG_BYTES) return false;
-  if (file === RUNTIME_CATALOG_FILE && info.uid === 0 && (mode === 0o444 || mode === 0o644)) return true;
-  return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600 || mode === 0o644);
-}
-
-function readProtectedCatalog(file: string): unknown {
-  let descriptor: number | undefined;
-  try {
-    const before = lstatSync(file);
-    if (!protectedCatalogStat(file, before)) throw new Error("runtime model catalog is unavailable");
-    descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
-    const opened = fstatSync(descriptor);
-    if (!protectedCatalogStat(file, opened)
-      || before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("runtime model catalog is unavailable");
-    const source = readFileSync(descriptor, "utf8");
-    const after = fstatSync(descriptor);
-    const current = lstatSync(file);
-    if (!protectedCatalogStat(file, after) || !protectedCatalogStat(file, current)
-      || opened.dev !== after.dev || opened.ino !== after.ino
-      || opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("runtime model catalog is unavailable");
-    return JSON.parse(source);
-  } catch {
-    throw new Error("runtime model catalog is unavailable");
-  } finally {
-    if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized above */ }
-  }
-}
-
-export function loadRuntimeModelCatalog(file?: string): RuntimeModelCatalog {
-  if (!file) return new RestartLoadedRuntimeModelCatalog();
-  const parsed = catalogSchema.safeParse(readProtectedCatalog(file));
-  if (!parsed.success) throw new Error("runtime model catalog is invalid");
-  if (parsed.data.models.some((model) => !validRuntimeModel(model))) {
-    throw new Error("runtime model catalog is invalid");
-  }
-  const ids = new Set(parsed.data.models.map((model) => model.id));
-  if (ids.size !== parsed.data.models.length) throw new Error("runtime model catalog contains duplicate models");
-  const sessions = parsed.data.models.filter((model) => model.session !== undefined).map((model) => model.id);
-  const metadata = parsed.data.models.filter((model) => model.metadataGeneration !== undefined).map((model) => model.id);
-  if (!sessions.includes(parsed.data.defaultSession)) throw new Error("runtime model catalog session default is invalid");
-  if ((metadata.length > 0) !== (parsed.data.defaultMetadataGeneration !== undefined)
-    || (parsed.data.defaultMetadataGeneration !== undefined
-      && !metadata.includes(parsed.data.defaultMetadataGeneration))) {
-    throw new Error("runtime model catalog metadata default is invalid");
-  }
-  return new RestartLoadedRuntimeModelCatalog(parsed.data);
-}
-
-function validRuntimeModel(model: RuntimeModel): boolean {
-  if ((model.session !== undefined) !== (model.sessionAdapter !== undefined)) return false;
-  if ((model.metadataGeneration !== undefined) !== (model.metadataAdapter !== undefined)) return false;
-  switch (model.authentication.mode) {
-    case "secret_env":
-      return model.authentication.apiKeyEnv !== undefined
-        && secretBundleKey.test(model.authentication.apiKeyEnv);
-    case "pi_auth":
-      return model.authentication.apiKeyEnv === undefined
-        && model.metadataGeneration === undefined
-        && model.sessionAdapter?.mode === "pi_builtin";
-    case "none":
-      return model.authentication.apiKeyEnv === undefined && model.endpoint !== undefined;
-  }
-}
-
-export function splitCanonicalModelId(id: string): { provider: string; model: string } {
-  const slash = id.indexOf("/");
-  if (slash <= 0 || slash === id.length - 1) throw new Error("model identity is invalid");
-  return { provider: id.slice(0, slash), model: id.slice(slash + 1) };
-}

backend/src/operator-command.ts

@@ -8,8 +8,8 @@ import { join } from "node:path";
 import { loadConfig, type AppConfig } from "./config.js";
 import { rolesToPermissions } from "./auth/config.js";
 import type { PrincipalContext } from "./auth/principal.js";
+import { createPiModelLister } from "./pi/list-models.js";
 import { createPiManagement } from "./pi/management.js";
-import { loadRuntimeModelCatalog } from "./models/runtime-model-catalog.js";
 import { effectiveSettings } from "./routes/settings.js";
 import { MaintenanceBarrier } from "./runtime/maintenance-gate.js";
 import { loadSettings } from "./settings/settings-store.js";
@@ -19,7 +19,7 @@ import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
 
 type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status"
   | "session-inventory" | "workflow-doctor" | "workspace-integrity"
-  | "pi-test" | "effective-settings";
+  | "pi-options" | "pi-test" | "effective-settings";
 
 const lifecyclePrincipal: PrincipalContext = {
   issuer: "tht-operator-command",
@@ -110,11 +110,9 @@ export async function runOperatorAction(
   if (action === "session-inventory") return await sessionInventory(config);
   if (action === "workflow-doctor") return await workflowDiagnostics(config);
   if (action === "workspace-integrity") return await workspaceIntegrity(config);
-  const modelCatalog = loadRuntimeModelCatalog(config.modelCatalogFile);
-  if (action === "effective-settings") {
-    return effectiveSettings(config, loadSettings(config), modelCatalog);
-  }
-  const service = createPiManagement(config, { modelCatalog });
+  if (action === "effective-settings") return effectiveSettings(config, loadSettings(config));
+  const service = createPiManagement(config, { listModels: createPiModelLister(config) });
+  if (action === "pi-options") return await service.options();
   if (action === "pi-test") return await service.test();
   throw new Error("unsupported operator action");
 }
@@ -123,7 +121,7 @@ async function main(): Promise<void> {
   const action = process.argv[2] as OperatorAction | undefined;
   if (!action || ![
     "maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory",
-    "workflow-doctor", "workspace-integrity", "pi-test", "effective-settings",
+    "workflow-doctor", "workspace-integrity", "pi-options", "pi-test", "effective-settings",
   ].includes(action)) throw new Error("invalid operator action");
   const result = await runOperatorAction(action, loadConfig(process.env));
   process.stdout.write(`${JSON.stringify(result)}\n`);

backend/src/pi/list-models.ts

@@ -10,7 +10,6 @@ import {
   readConfiguredPiAgentFile,
   validateDeclarativePiConfig,
 } from "./managed-config.js";
-import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
 
 export interface PiModel {
   provider: string;
@@ -19,8 +18,6 @@ export interface PiModel {
   reasoning: boolean;
 }
 
-export type ListModelsFn = () => Promise<PiModel[]>;
-
 interface Opts {
   spawnFn?: (
     command: string,
@@ -31,7 +28,6 @@ interface Opts {
   nowMs?: () => number;
   loadEnabledModels?: () => PiEnabledModelsResult;
   readModelsStore?: () => string | undefined;
-  modelCatalog?: RuntimeModelCatalog;
   warn?: (message: string) => void;
 }
 
@@ -40,7 +36,7 @@ interface Opts {
  * configured) via an ephemeral `pi --mode rpc` process. Result is cached for
  * `ttlMs`. The returned function rejects on timeout/error; callers degrade.
  */
-export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModelsFn {
+export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): () => Promise<PiModel[]> {
   const ttlMs = opts.ttlMs ?? 60_000;
   const now = opts.nowMs ?? (() => Date.now());
   const spawnFn = opts.spawnFn ?? nodeSpawn;
@@ -84,23 +80,9 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModels
       const byCompositeId = new Map(
         available.map((model) => [`${model.provider}/${model.id}`, model]),
       );
-      const catalogByPiId = new Map(
-        (opts.modelCatalog?.sessionModels() ?? []).map((model) => [
-          `${model.provider}/${model.upstreamModel}`,
-          model,
-        ]),
-      );
       const models = enabled.ids.flatMap((id) => {
         const model = byCompositeId.get(id);
-        if (!model) return [];
-        const catalogModel = catalogByPiId.get(id);
-        return [catalogModel ? {
-          ...model,
-          provider: catalogModel.provider,
-          id: catalogModel.model,
-          name: catalogModel.label,
-          reasoning: catalogModel.session?.reasoning ?? model.reasoning,
-        } : model];
+        return model ? [model] : [];
       });
       if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available");
       cache = { at: now(), models };

backend/src/pi/management.ts

@@ -2,11 +2,12 @@ import { execFile as nodeExecFile } from "node:child_process";
 import { promisify } from "node:util";
 import type { AppConfig } from "../config.js";
 import { secretValue } from "../config/secret-bundle.js";
-import { loadSettings, type Settings } from "../settings/settings-store.js";
 import {
-  splitCanonicalModelId,
-  type RuntimeModelCatalog,
-} from "../models/runtime-model-catalog.js";
+  loadSettings,
+  saveSettings,
+  type Settings,
+} from "../settings/settings-store.js";
+import type { PiModel } from "./list-models.js";
 import {
   configuredPiProviderApiKey,
   PI_MANAGED_CONFIG_ERROR_MESSAGE,
@@ -44,6 +45,13 @@ export interface PiStatus {
   message?: string;
 }
 
+export interface PiOptions {
+  providers: string[];
+  models: Array<{ provider: string; id: string }>;
+  reasoning: PiReasoning[];
+  checkedAt: string;
+}
+
 export interface PiTestResult {
   ready: boolean;
   checkedAt: string;
@@ -68,13 +76,15 @@ export type PiExecFile = (
 
 export interface PiManagementService {
   status(): Promise<PiStatus>;
+  options(): Promise<PiOptions>;
+  configure(value: PiInstallationConfig): Promise<PiInstallationConfig & { updatedAt: string }>;
   test(): Promise<PiTestResult>;
   logs(): Promise<PiLogs>;
 }
 
 export class PiManagementError extends Error {
   constructor(
-    public readonly code: "pi_management_unavailable",
+    public readonly code: "pi_management_invalid_config" | "pi_management_unavailable" | "pi_management_write_failed",
     message: string,
   ) {
     super(message);
@@ -83,9 +93,10 @@ export class PiManagementError extends Error {
 
 interface PiManagementDeps {
   execute?: PiExecFile;
-  modelCatalog: RuntimeModelCatalog;
+  listModels: () => Promise<PiModel[]>;
   smokeProvider?: PiProviderSmoke;
   readSettings?: () => Settings;
+  saveSettings?: (settings: Settings) => Settings;
   readLogs?: () => string | Promise<string>;
   credentialStatus?: (provider: string | undefined) => PiCredentialStatus;
   now?: () => Date;
@@ -100,36 +111,54 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
   };
   const execute = deps.execute ?? defaultExecFile;
   const readSettings = deps.readSettings ?? (() => loadSettings(config));
+  const persistSettings = deps.saveSettings ?? ((settings) => saveSettings(config, settings));
   const readLogs = deps.readLogs ?? (() => diagnostics.join("\n"));
-  const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config, {
-    modelCatalog: deps.modelCatalog,
-  });
+  const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config);
   const credentialStatus = deps.credentialStatus ?? ((provider: string | undefined) => {
     try {
-      const model = deps.modelCatalog.defaultSession
-        ? deps.modelCatalog.sessionModels().find((entry) => entry.id === deps.modelCatalog.defaultSession)
-        : undefined;
-      const credentialName = model?.authentication.mode === "secret_env"
-        ? model.authentication.apiKeyEnv
-        : undefined;
-      const configuredApiKey = configuredPiProviderApiKey(
-        readConfiguredPiAgentFile("models.json", true),
-        provider,
-      ) ?? (credentialName ? `$${credentialName}` : undefined);
       return piProviderCredentialStatus({
         provider,
         authProviders: loadPiAuthProviders(),
-        resolveCredentialValue: () => credentialName
-          ? secretValue(config, credentialName)
-          : config.modelCatalogFile ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
+        resolveCredentialValue: () => secretValue(config, "THT_MODEL_API_KEY"),
         credentialFile: config.modelApiKeyFile,
-        configuredApiKey,
+        configuredApiKey: configuredPiProviderApiKey(
+          readConfiguredPiAgentFile("models.json", true),
+          provider,
+        ),
       });
     } catch {
       return "missing";
     }
   });
 
+  const closedOptions = async (): Promise<Omit<PiOptions, "checkedAt">> => {
+    let listed: PiModel[];
+    try {
+      listed = await deps.listModels();
+    } catch (error) {
+      if (isPiManagedConfigError(error)) {
+        throw new PiManagementError("pi_management_unavailable", PI_MANAGED_CONFIG_ERROR_MESSAGE);
+      }
+      throw new PiManagementError("pi_management_unavailable", "Pi model choices are unavailable");
+    }
+    const models: Array<{ provider: string; id: string }> = [];
+    const providers: string[] = [];
+    const seenModels = new Set<string>();
+    const seenProviders = new Set<string>();
+    for (const model of listed) {
+      if (!isChoice(model?.provider) || !isChoice(model?.id)) continue;
+      const key = `${model.provider}\u0000${model.id}`;
+      if (seenModels.has(key)) continue;
+      seenModels.add(key);
+      models.push({ provider: model.provider, id: model.id });
+      if (!seenProviders.has(model.provider)) {
+        seenProviders.add(model.provider);
+        providers.push(model.provider);
+      }
+    }
+    return { providers, models, reasoning: [...REASONING_CHOICES] };
+  };
+
   const version = async (timeoutMs = config.piManagementTimeoutMs): Promise<string> => {
     let output: { stdout: string; stderr: string };
     try {
@@ -151,12 +180,12 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
 
   const installationConfig = (): PiInstallationConfig => {
     const settings = readSettings();
+    const provider = config.defaults.provider ?? settings.provider;
+    const model = config.defaults.model ?? settings.model;
     const reasoning = config.defaults.thinking ?? settings.thinking;
-    const selected = deps.modelCatalog.defaultSession
-      ? splitCanonicalModelId(deps.modelCatalog.defaultSession)
-      : undefined;
     return {
-      ...(selected ? selected : {}),
+      ...(isChoice(provider) ? { provider } : {}),
+      ...(isChoice(model) ? { model } : {}),
       ...(isReasoning(reasoning) ? { reasoning } : {}),
     };
   };
@@ -177,6 +206,28 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
       }
     },
 
+    async options(): Promise<PiOptions> {
+      const choices = await closedOptions();
+      return { ...choices, checkedAt: now().toISOString() };
+    },
+
+    async configure(value: PiInstallationConfig): Promise<PiInstallationConfig & { updatedAt: string }> {
+      if (!isInstallationConfig(value)) {
+        throw new PiManagementError("pi_management_invalid_config", "Pi installation configuration is invalid");
+      }
+      const choices = await closedOptions();
+      if (!choices.models.some((model) => model.provider === value.provider && model.id === value.model)) {
+        throw new PiManagementError("pi_management_invalid_config", "Pi provider and model must be selected from available choices");
+      }
+      try {
+        persistSettings({ ...readSettings(), provider: value.provider, model: value.model, thinking: value.reasoning });
+      } catch {
+        throw new PiManagementError("pi_management_write_failed", "Pi installation configuration could not be saved");
+      }
+      addDiagnostic("Pi installation defaults updated");
+      return { ...value, updatedAt: now().toISOString() };
+    },
+
     async test(): Promise<PiTestResult> {
       const checkedAt = now().toISOString();
       const deadline = Date.now() + config.piManagementTimeoutMs;
@@ -242,10 +293,24 @@ async function defaultExecFile(command: string, args: string[], options: PiExecF
   return { stdout: String(result.stdout), stderr: String(result.stderr) };
 }
 
+function isChoice(value: unknown): value is string {
+  return typeof value === "string" && value.length > 0 && value.length <= 128 && value.trim() === value
+    && /^[A-Za-z0-9][A-Za-z0-9._/-]*$/u.test(value);
+}
+
 function isReasoning(value: unknown): value is PiReasoning {
   return typeof value === "string" && (REASONING_CHOICES as readonly string[]).includes(value);
 }
 
+function isInstallationConfig(value: unknown): value is Required<PiInstallationConfig> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const candidate = value as Record<string, unknown>;
+  if (Object.keys(candidate).length !== 3 || Object.keys(candidate).some((key) => !["provider", "model", "reasoning"].includes(key))) {
+    return false;
+  }
+  return isChoice(candidate.provider) && isChoice(candidate.model) && isReasoning(candidate.reasoning);
+}
+
 function isTimeout(error: unknown): boolean {
   return Boolean(
     error && typeof error === "object" && (

backend/src/pi/pi-process-manager.ts

@@ -11,10 +11,6 @@ import {
   configuredPiProviderApiKey,
   createPiRuntimeAgentSnapshot,
 } from "./managed-config.js";
-import {
-  loadRuntimeModelCatalog,
-  type RuntimeModelCatalog,
-} from "../models/runtime-model-catalog.js";
 
 export interface SessionRuntime {
   rpc: RpcClient;
@@ -46,32 +42,23 @@ export class PiProcessManager {
   private runtimes = new Map<string, SessionRuntime>();
   private agentSnapshotCleanups = new WeakMap<ChildProcessWithoutNullStreams, () => void>();
   private spawnFn: (
-    sessionId: string, author: string, provider: string | undefined, model: string | undefined,
-    principal?: PrincipalContext, runtimeConfigPath?: string,
+    sessionId: string, author: string, provider: string | undefined, principal?: PrincipalContext,
+    runtimeConfigPath?: string,
   ) => ChildProcessWithoutNullStreams;
   private loadAuthProviders: (agentDir: string) => ReadonlySet<string>;
-  private modelCatalog: RuntimeModelCatalog;
-  private modelCatalogConfigured: boolean;
 
   constructor(
     private cfg: AppConfig,
-    opts?: {
-      spawnFn?: SpawnFn;
-      authProviders?: (agentDir: string) => ReadonlySet<string>;
-      modelCatalog?: RuntimeModelCatalog;
-    },
+    opts?: { spawnFn?: SpawnFn; authProviders?: (agentDir: string) => ReadonlySet<string> },
   ) {
-    this.modelCatalog = opts?.modelCatalog ?? loadRuntimeModelCatalog(cfg.modelCatalogFile);
-    this.modelCatalogConfigured = cfg.modelCatalogFile !== undefined
-      || this.modelCatalog.defaultSession !== null;
     this.loadAuthProviders = opts?.authProviders
       ?? ((agentDir) => loadPiAuthProviders({ agentDir }));
     if (opts?.spawnFn) {
-      this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
-        this.spawnPi(opts.spawnFn!, sessionId, author, provider, model, principal, runtimeConfigPath);
+      this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) =>
+        this.spawnPi(opts.spawnFn!, sessionId, author, provider, principal, runtimeConfigPath);
     } else {
-      this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
-        this.spawnPi(nodeSpawn, sessionId, author, provider, model, principal, runtimeConfigPath);
+      this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) =>
+        this.spawnPi(nodeSpawn, sessionId, author, provider, principal, runtimeConfigPath);
     }
   }
 
@@ -84,7 +71,7 @@ export class PiProcessManager {
 
   private spawnPi(
     spawnFn: SpawnFn, sessionId: string, author: string, provider: string | undefined,
-    model: string | undefined, principal?: PrincipalContext, runtimeConfigPath?: string,
+    principal?: PrincipalContext, runtimeConfigPath?: string,
   ): ChildProcessWithoutNullStreams {
     // This is the final shared boundary for createFor(), spawnFor(), and resume(). Validate
     // before auth-provider inspection, then make Pi consume the exact copied bytes rather than
@@ -92,23 +79,12 @@ export class PiProcessManager {
     const agent = createPiRuntimeAgentSnapshot();
     let child: ChildProcessWithoutNullStreams | undefined;
     try {
-      const catalogModel = provider && model
-        ? this.modelCatalog.sessionModels()
-          .find((entry) => entry.provider === provider && entry.model === model)
-        : undefined;
-      const credentialName = catalogModel?.authentication.mode === "secret_env"
-        ? catalogModel.authentication.apiKeyEnv
-        : undefined;
-      const projectedApiKey = configuredPiProviderApiKey(agent.models, provider)
-        ?? (credentialName ? `$${credentialName}` : undefined);
       const env = buildPiChildEnv({
         provider,
         authProviders: this.loadAuthProviders(agent.agentDir),
-        credentialValue: credentialName
-          ? secretValue(this.cfg, credentialName)
-          : this.modelCatalogConfigured ? undefined : secretValue(this.cfg, "THT_MODEL_API_KEY"),
+        credentialValue: secretValue(this.cfg, "THT_MODEL_API_KEY"),
         credentialFile: this.cfg.modelApiKeyFile,
-        configuredApiKey: projectedApiKey,
+        configuredApiKey: configuredPiProviderApiKey(agent.models, provider),
         additions: { THT_SESSION: sessionId, THT_AUTHOR: author },
       });
       env.PI_CODING_AGENT_DIR = agent.agentDir;
@@ -186,10 +162,9 @@ export class PiProcessManager {
     }
     const author = o.author ?? "dev@local";
     const provider = canonicalPiProvider(o.provider ?? this.cfg.defaults.provider);
-    const model = o.model ?? this.cfg.defaults.model;
     let child: ChildProcessWithoutNullStreams;
     try {
-      child = this.spawnFn(sessionId, author, provider, model, o.principal, o.runtimeConfig?.path);
+      child = this.spawnFn(sessionId, author, provider, o.principal, o.runtimeConfig?.path);
     } catch (error) {
       o.runtimeConfig?.release();
       throw error;
@@ -260,11 +235,8 @@ export class PiProcessManager {
     const thinking = o.thinking ?? this.cfg.defaults.thinking;
 
     if (provider && model) {
-      const upstreamModel = this.modelCatalog.sessionModels()
-        .find((entry) => entry.provider === provider && entry.model === model)
-        ?.upstreamModel ?? model;
       const response = await rt.rpc.request(
-        { type: "set_model", provider, modelId: upstreamModel } as object & { type: string },
+        { type: "set_model", provider, modelId: model } as object & { type: string },
       );
       rt.bridge.setContextWindow(response?.data?.contextWindow);
     }

backend/src/pi/provider-smoke.ts

@@ -17,10 +17,6 @@ import {
   readConfiguredPiAgentFile,
   validateDeclarativePiConfig,
 } from "./managed-config.js";
-import {
-  loadRuntimeModelCatalog,
-  type RuntimeModelCatalog,
-} from "../models/runtime-model-catalog.js";
 
 const SMOKE_PROMPT = "Provider health check. Reply with exactly OK.";
 const SMOKE_ARGS = [
@@ -53,7 +49,6 @@ interface ProviderSmokeOptions {
   authProviders?: () => ReadonlySet<string>;
   readAuthStore?: () => string;
   readModelsStore?: () => string | undefined;
-  modelCatalog?: RuntimeModelCatalog;
 }
 
 export function createPiProviderSmoke(
@@ -74,25 +69,12 @@ export function createPiProviderSmoke(
       const configuredModels = options.readModelsStore
         ? options.readModelsStore()
         : readConfiguredPiAgentFile("models.json", true);
-      const catalog = options.modelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
-      const catalogConfigured = config.modelCatalogFile !== undefined
-        || catalog.defaultSession !== null;
-      const catalogModel = catalog.sessionModels()
-        .find((entry) => entry.provider === canonicalProvider && entry.model === model);
-      const upstreamModel = catalogModel?.upstreamModel ?? model;
-      const credentialName = catalogModel?.authentication.mode === "secret_env"
-        ? catalogModel.authentication.apiKeyEnv
-        : undefined;
-      const projectedApiKey = configuredPiProviderApiKey(configuredModels, canonicalProvider)
-        ?? (credentialName ? `$${credentialName}` : undefined);
       const env = buildPiChildEnv({
         provider: canonicalProvider,
         authProviders: configuredAuthProviders,
-        credentialValue: credentialName
-          ? secretValue(config, credentialName)
-          : catalogConfigured ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
+        credentialValue: secretValue(config, "THT_MODEL_API_KEY"),
         credentialFile: config.modelApiKeyFile,
-        configuredApiKey: projectedApiKey,
+        configuredApiKey: configuredPiProviderApiKey(configuredModels, canonicalProvider),
       });
       clearPrincipalEnvironment(env);
       delete env.THT_DATA_ROOT;
@@ -131,7 +113,7 @@ export function createPiProviderSmoke(
       const capabilityGuard = failOnUnexpectedCapabilities(rpc);
       const turn = async (): Promise<void> => {
         requireSuccessfulResponse(await rpc.request({
-          type: "set_model", provider: canonicalProvider, modelId: upstreamModel,
+          type: "set_model", provider: canonicalProvider, modelId: model,
         } as object & { type: string }));
         requireSuccessfulResponse(await rpc.request({
           type: "set_thinking_level", level: reasoning,

backend/src/routes/catalog-description-consolidation.ts

@@ -9,13 +9,10 @@ import {
 } from "../catalog/types.js";
 
 const idSchema = z.uuid();
-const consolidationSchema = z.discriminatedUnion("target", [
-  z.object({
-    target: z.enum(["tables", "columns"]),
-    targetIds: z.array(idSchema).min(1).max(10_000),
-  }).strict(),
-  z.object({ target: z.enum(["database", "database_columns"]) }).strict(),
-]);
+const consolidationSchema = z.object({
+  target: z.enum(["tables", "columns"]),
+  targetIds: z.array(idSchema).min(1).max(10_000),
+}).strict();
 
 function manage(request: FastifyRequest, reply: FastifyReply) {
   return isPrincipalContext(requirePermission(request, reply, "database.manage"));
@@ -52,7 +49,7 @@ export function catalogDescriptionConsolidationRoutes(
     try {
       const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
       const input = consolidationSchema.parse(request.body);
-      const targetIds = "targetIds" in input ? [...new Set(input.targetIds)] : [];
+      const targetIds = [...new Set(input.targetIds)];
       const result = await deps.operations.run(
         databaseId,
         async () => await deps.repository.consolidateGeneratedDescriptions(

backend/src/routes/catalog-description-generation.ts

@@ -11,35 +11,38 @@ import {
   type DescriptionGenerationWorker,
 } from "../catalog/description-generation-worker.js";
 import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js";
+import { ModelCompletionProviderError } from "../catalog/model-completer.js";
 import {
-  SensitivityAnalysisDuplicateTargetIdsError,
-  SensitivityAnalysisInterruptedError,
-  SensitivityAnalysisNoEligibleColumnsError,
-  SensitivityAnalysisTargetNotFoundError,
-} from "../catalog/sensitivity-analysis-service.js";
-import type { SensitivityAnalysisRunner } from "../catalog/sensitivity-analysis-runner.js";
+  SensitiveDataSuggestionDuplicateTargetIdsError,
+  SensitiveDataSuggestionInvalidResponseError,
+  SensitiveDataSuggestionNoEligibleColumnsError,
+  SensitiveDataSuggestionPayloadTooLargeError,
+  SensitiveDataSuggestionTargetNotFoundError,
+} from "../catalog/sensitive-data-suggester.js";
+import type { SensitiveDataSuggestionRunner } from "../catalog/sensitive-data-suggestion-runner.js";
 import {
   CatalogOperationInProgressError,
-  CatalogConnectorError,
   CatalogUnavailableError,
   DescriptionGenerationRunActiveError,
   type CatalogRepository,
   type DescriptionGenerationEvent,
   type DescriptionGenerationRun,
-  type SensitivityAnalysisEvent,
-  type SensitivityAnalysisRun,
+  type SensitiveDataSuggestionEvent,
+  type SensitiveDataSuggestionRun,
 } from "../catalog/types.js";
 
 const idSchema = z.uuid();
-const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/);
+const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
 const selectedTargetIdsSchema = z.array(idSchema).min(1);
 const suggestionSchema = z.discriminatedUnion("scope", [
-  z.object({ scope: z.literal("all") }).strict(),
+  z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
   z.object({
+    modelId: modelIdSchema,
     scope: z.literal("selected_tables"),
     targetIds: selectedTargetIdsSchema,
   }).strict(),
   z.object({
+    modelId: modelIdSchema,
     scope: z.literal("selected_columns"),
     targetIds: selectedTargetIdsSchema,
   }).strict(),
@@ -109,7 +112,7 @@ function publicRun(run: DescriptionGenerationRun) {
   };
 }
 
-function publicSensitivityAnalysisEvent(event: SensitivityAnalysisEvent) {
+function publicSensitiveDataSuggestionEvent(event: SensitiveDataSuggestionEvent) {
   return {
     runId: event.runId,
     sequence: event.sequence,
@@ -119,19 +122,16 @@ function publicSensitivityAnalysisEvent(event: SensitivityAnalysisEvent) {
   };
 }
 
-function publicSensitivityAnalysisRun(run: SensitivityAnalysisRun) {
+function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) {
   return {
     id: run.id,
     databaseId: run.databaseId,
     scope: run.scope,
-    engine: run.engine,
     modelId: run.modelId,
-    policyVersion: run.policyVersion,
     status: run.status,
     total: run.total,
     suggestedSensitive: run.suggestedSensitive,
     suggestedNonSensitive: run.suggestedNonSensitive,
-    unknown: run.unknown,
     inputTokens: run.inputTokens,
     cacheReadTokens: run.cacheReadTokens,
     outputTokens: run.outputTokens,
@@ -232,10 +232,16 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
   if (error instanceof CatalogUnavailableError) {
     return reply.code(503).send({
       code: "catalog_unavailable",
-      message: "The database catalog is unavailable, so no sensitivity assessments were prepared.",
+      message: "The database catalog is unavailable, so no sensitive-field suggestions were prepared.",
     });
   }
-  if (error instanceof SensitivityAnalysisTargetNotFoundError) {
+  if (error instanceof MetadataGenerationModelUnavailableError) {
+    return reply.code(409).send({
+      code: "metadata_generation_model_unavailable",
+      message: "The selected metadata-generation model is unavailable.",
+    });
+  }
+  if (error instanceof SensitiveDataSuggestionTargetNotFoundError) {
     const code = error.target === "database"
       ? "database_not_found"
       : error.target === "table"
@@ -248,39 +254,45 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
         : "One or more selected Catalog Columns were not found in this database.";
     return reply.code(404).send({ code, message });
   }
-  if (error instanceof SensitivityAnalysisDuplicateTargetIdsError) {
+  if (error instanceof SensitiveDataSuggestionDuplicateTargetIdsError) {
     return reply.code(400).send({
       code: "sensitive_data_suggestion_target_ids_duplicate",
       message: "Each selected table or column must appear only once.",
     });
   }
-  if (error instanceof SensitivityAnalysisNoEligibleColumnsError) {
+  if (error instanceof SensitiveDataSuggestionNoEligibleColumnsError) {
     return reply.code(409).send({
       code: "sensitive_data_suggestion_no_columns",
-      message: "The selected scope contains no Catalog Columns to assess.",
+      message: "The selected scope contains no Catalog Columns to classify.",
     });
   }
-  if (error instanceof SensitivityAnalysisInterruptedError) {
-    return reply.code(499).send({
-      code: "sensitivity_analysis_interrupted",
-      message: "Sensitivity analysis was interrupted before completion. No assessments were applied.",
+  if (error instanceof SensitiveDataSuggestionPayloadTooLargeError) {
+    return reply.code(413).send({
+      code: "sensitive_data_suggestion_payload_too_large",
+      message: "The selected structural metadata cannot be divided into safe LLM requests.",
     });
   }
-  if (error instanceof CatalogConnectorError) {
+  if (error instanceof SensitiveDataSuggestionInvalidResponseError) {
     return reply.code(502).send({
-      code: "sensitivity_source_unavailable",
-      message: "The source values could not be inspected safely. No assessments were applied.",
+      code: "sensitive_data_suggestion_invalid_response",
+      message: "The LLM returned an incomplete or invalid classification. No suggestions were applied.",
+    });
+  }
+  if (error instanceof ModelCompletionProviderError) {
+    return reply.code(502).send({
+      code: "sensitive_data_suggestion_provider_unavailable",
+      message: "The selected LLM service could not complete the request. No suggestions were applied.",
     });
   }
   if (error instanceof z.ZodError) {
     return reply.code(400).send({
       code: "sensitive_data_suggestion_request_invalid",
-      message: "Choose a database, one or more tables, or one or more columns to assess.",
+      message: "Choose a database, one or more tables, or one or more columns to classify.",
     });
   }
   return reply.code(500).send({
     code: "sensitive_data_suggestion_failed",
-    message: "Local sensitivity analysis failed before review. No changes were applied.",
+    message: "Sensitive-field suggestions failed before review. No changes were applied.",
   });
 }
 
@@ -288,18 +300,18 @@ function safeSuggestionHistoryError(reply: FastifyReply, error: unknown) {
   if (error instanceof CatalogUnavailableError) {
     return reply.code(503).send({
       code: "catalog_unavailable",
-      message: "Sensitivity Analysis history is unavailable because the database catalog is unavailable.",
+      message: "Sensitive Data Suggestion history is unavailable because the database catalog is unavailable.",
     });
   }
   if (error instanceof z.ZodError) {
     return reply.code(400).send({
       code: "sensitive_data_suggestion_history_request_invalid",
-      message: "Sensitivity Analysis history parameters are invalid.",
+      message: "Sensitive Data Suggestion history parameters are invalid.",
     });
   }
   return reply.code(500).send({
     code: "sensitive_data_suggestion_history_failed",
-    message: "Sensitivity Analysis history could not be loaded.",
+    message: "Sensitive Data Suggestion history could not be loaded.",
   });
 }
 
@@ -308,7 +320,7 @@ export function catalogDescriptionGenerationRoutes(
   deps: {
     repository: CatalogRepository;
     worker: DescriptionGenerationWorker;
-    sensitivityAnalysisRunner: SensitivityAnalysisRunner;
+    sensitiveDataSuggestionRunner: SensitiveDataSuggestionRunner;
   },
 ): void {
   app.post("/catalog/databases/:databaseId/sensitive-data-suggestions", async (request, reply) => {
@@ -316,25 +328,16 @@ export function catalogDescriptionGenerationRoutes(
     try {
       const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
       const input = suggestionSchema.parse(request.body);
-      const controller = new AbortController();
-      const abort = () => controller.abort();
-      request.raw.once("aborted", abort);
-      reply.raw.once("close", abort);
-      let result;
-      try {
-        result = await deps.sensitivityAnalysisRunner.run(
-          databaseId,
-          input.scope,
-          "targetIds" in input ? input.targetIds : [],
-          controller.signal,
-        );
-      } finally {
-        request.raw.off("aborted", abort);
-        reply.raw.off("close", abort);
-      }
+      const result = await deps.sensitiveDataSuggestionRunner.run(
+        databaseId,
+        input.modelId,
+        input.scope,
+        "targetIds" in input ? input.targetIds : [],
+        new AbortController().signal,
+      );
       return {
         suggestions: result.suggestions,
-        run: publicSensitivityAnalysisRun(result.run),
+        run: publicSensitiveDataSuggestionRun(result.run),
       };
     } catch (error) {
       return safeSuggestionError(reply, error);
@@ -345,8 +348,8 @@ export function catalogDescriptionGenerationRoutes(
     if (!manage(request, reply)) return reply;
     try {
       const { limit } = historyQuerySchema.parse(request.query);
-      return (await deps.repository.listSensitivityAnalysisRuns(limit))
-        .map(publicSensitivityAnalysisRun);
+      return (await deps.repository.listSensitiveDataSuggestionRuns(limit))
+        .map(publicSensitiveDataSuggestionRun);
     } catch (error) {
       return safeSuggestionHistoryError(reply, error);
     }
@@ -356,12 +359,12 @@ export function catalogDescriptionGenerationRoutes(
     if (!manage(request, reply)) return reply;
     try {
       const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
-      const run = await deps.repository.getSensitivityAnalysisRun(runId);
+      const run = await deps.repository.getSensitiveDataSuggestionRun(runId);
       if (!run) return reply.code(404).send({
         code: "sensitive_data_suggestion_run_not_found",
-        message: "Sensitivity Analysis Run was not found.",
+        message: "Sensitive Data Suggestion Run was not found.",
       });
-      return publicSensitivityAnalysisRun(run);
+      return publicSensitiveDataSuggestionRun(run);
     } catch (error) {
       return safeSuggestionHistoryError(reply, error);
     }
@@ -372,14 +375,14 @@ export function catalogDescriptionGenerationRoutes(
     try {
       const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
       const { after } = eventQuerySchema.parse(request.query);
-      if (!(await deps.repository.getSensitivityAnalysisRun(runId))) {
+      if (!(await deps.repository.getSensitiveDataSuggestionRun(runId))) {
         return reply.code(404).send({
           code: "sensitive_data_suggestion_run_not_found",
-          message: "Sensitivity Analysis Run was not found.",
+          message: "Sensitive Data Suggestion Run was not found.",
         });
       }
-      return (await deps.repository.listSensitivityAnalysisEvents(runId, after))
-        .map(publicSensitivityAnalysisEvent);
+      return (await deps.repository.listSensitiveDataSuggestionEvents(runId, after))
+        .map(publicSensitiveDataSuggestionEvent);
     } catch (error) {
       return safeSuggestionHistoryError(reply, error);
     }

backend/src/routes/catalog-schema.ts

@@ -19,14 +19,8 @@ const metadataSchema = z.object({
   description: z.string().max(20_000).nullable().optional(),
   generatedDescription: z.string().max(20_000).nullable().optional(),
   sensitive: z.boolean().optional(),
-  sensitivityReason: z.string().max(2_000).nullable().optional(),
 }).strict().refine((value) => (
-  "description" in value
-  || "generatedDescription" in value
-  || "sensitive" in value
-  || "sensitivityReason" in value
-)).refine((value) => (
-  value.sensitivityReason == null || value.sensitive === true
+  "description" in value || "generatedDescription" in value || "sensitive" in value
 ));
 const createRunSchema = z.object({
   version: z.number().int().positive(),
@@ -62,10 +56,7 @@ function safeError(reply: FastifyReply, error: unknown) {
     return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
   }
   if (error instanceof CatalogConnectorError) {
-    return reply.code(502).send({
-      code: "schema_introspection_failed",
-      message: "The database schema could not be read. Check the connection and credentials, then try again.",
-    });
+    return reply.code(502).send({ code: "schema_introspection_failed", message: "The database schema could not be read safely." });
   }
   if (error instanceof z.ZodError) {
     return reply.code(400).send({ code: "schema_request_invalid", message: "Schema request is invalid." });
@@ -122,12 +113,6 @@ export function catalogSchemaRoutes(
         if (current.version !== input.version) {
           return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
         }
-        const nextSensitive = input.sensitive ?? current.sensitive;
-        const nextSensitivityReason = nextSensitive
-          ? ("sensitivityReason" in input
-              ? normalized(input.sensitivityReason ?? null)
-              : current.sensitivityReason)
-          : null;
         const updated = await deps.repository.updateColumnMetadata(
           databaseId,
           tableId,
@@ -138,7 +123,6 @@ export function catalogSchemaRoutes(
             ? normalized(input.generatedDescription ?? null)
             : current.generatedDescription,
           input.sensitive,
-          nextSensitivityReason,
         );
         if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
         return updated;

backend/src/routes/meta.ts

@@ -1,8 +1,10 @@
 import { readdirSync } from "node:fs";
 import { join } from "node:path";
 import type { FastifyInstance } from "fastify";
+import type { PiModel } from "../pi/list-models.js";
 import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
-import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
+
+export type ListModelsFn = () => Promise<PiModel[]>;
 
 /**
  * List YAML workspace configs found in <harnessDir>/workspaces/*.yaml.
@@ -23,17 +25,20 @@ export function listWorkspaces(harnessDir: string): { name: string; file: string
 
 export function metaRoutes(
   app: FastifyInstance,
-  deps: { harnessDir: string; modelCatalog: RuntimeModelCatalog },
+  deps: { harnessDir: string; listModels?: ListModelsFn },
 ): void {
   app.get("/models", async (request, reply) => {
     if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply;
-    return {
-      models: deps.modelCatalog.sessionModels().map((entry) => ({
-        provider: entry.provider,
-        id: entry.model,
-        name: entry.label,
-        reasoning: entry.session?.reasoning ?? false,
-      })),
-    };
+    const fn = deps.listModels ?? (async () => []);
+    try {
+      return { models: await fn() };
+    } catch (error) {
+      app.log.warn({
+        component: "pi-model-list",
+        errorType: error instanceof Error ? error.name : typeof error,
+      }, "Pi model listing failed");
+      // Graceful fallback: Pi may not be running; don't crash the server.
+      return { models: [] as PiModel[] };
+    }
   });
 }

backend/src/routes/pi-management.ts

@@ -11,6 +11,13 @@ export function piManagementRoutes(
   deps: { service: PiManagementService },
 ): void {
   app.get("/pi-management/status", async (request, reply) => run(request, reply, deps, () => deps.service.status()));
+  app.get("/pi-management/options", async (request, reply) => run(request, reply, deps, () => deps.service.options()));
+  app.put("/pi-management/config", async (request, reply) => run(
+    request,
+    reply,
+    deps,
+    () => deps.service.configure((request.body ?? {}) as Record<string, unknown>),
+  ));
   app.post("/pi-management/test", async (request, reply) => run(request, reply, deps, () => deps.service.test()));
   app.get("/pi-management/logs", async (request, reply) => run(request, reply, deps, () => deps.service.logs()));
 }
@@ -31,7 +38,8 @@ async function run<T>(
     return await action();
   } catch (error) {
     if (error instanceof PiManagementError) {
-      return reply.code(503).send({ code: error.code, error: error.message });
+      const statusCode = error.code === "pi_management_invalid_config" ? 400 : 503;
+      return reply.code(statusCode).send({ code: error.code, error: error.message });
     }
     return reply.code(503).send({ code: "pi_management_unavailable", error: "Pi management is unavailable" });
   }

backend/src/routes/sessions.ts

@@ -6,13 +6,12 @@ import type { Settings } from "../settings/settings-store.js";
 import { getPrincipal } from "../auth/auth.js";
 import type { PrincipalContext } from "../auth/principal.js";
 import type { ReadinessManager } from "../runtime/readiness-manager.js";
-import type { ListModelsFn } from "../pi/list-models.js";
+import type { ListModelsFn } from "./meta.js";
 import type { WorkspaceRegistry } from "../workspaces/registry.js";
 import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js";
 import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
 import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
-import { splitCanonicalModelId, type RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
-import type { CatalogRepository } from "../catalog/types.js";
+import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js";
 
 const BOOTSTRAP_FAILURE_MESSAGE =
   "Session startup failed. Check configuration and connectivity, then Resume the session.";
@@ -42,40 +41,11 @@ export function sessionRoutes(
     /** Fail-closed installation/runtime transport capability check. */
     workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean;
     maintenanceBarrier: MaintenanceBarrier;
-    modelCatalog: RuntimeModelCatalog;
-    /** PostgreSQL authority for mandatory preprocessing admission. */
-    catalogRepository?: CatalogRepository;
+    /** Optional only for narrow route-test stubs and installations without a Catalog database. */
+    effectiveRelationships?: EffectiveRelationshipSnapshotProvider;
   },
 ) {
   const lifecycleTails = new Map<string, Promise<void>>();
-
-  const preprocessingIsCurrent = async (
-    workspaceId: string,
-    inputFingerprint?: string,
-  ): Promise<boolean> => {
-    if (!d.catalogRepository) return true;
-    const database = await d.catalogRepository.getByWorkspace(workspaceId);
-    return database !== undefined
-      && database.preprocessingStatus === "succeeded"
-      && database.preprocessedMetadataRevision === database.metadataContentRevision
-      && (inputFingerprint === undefined
-        || database.preprocessingInputFingerprint === inputFingerprint);
-  };
-
-  const catalogTransportSupportsSessionRuntime = async (workspaceId: string): Promise<boolean> => {
-    if (!d.catalogRepository) return true;
-    const database = await d.catalogRepository.getByWorkspace(workspaceId);
-    return database === undefined || database.binding.transport !== "ssh_tunnel";
-  };
-
-  const currentInputFingerprint = async (
-    runner: any,
-    workspaceConfigPath: string,
-  ): Promise<string | undefined> => (
-    typeof runner.workspaceInputFingerprint === "function"
-      ? await runner.workspaceInputFingerprint(workspaceConfigPath)
-      : undefined
-  );
   const boundRuntimes = new Map<
     string,
     ReturnType<PiProcessManager["createFor"]>
@@ -120,13 +90,16 @@ export function sessionRoutes(
   const optionsWithRuntimeConfig = async (
     runner: any,
     workspaceConfigPath: string | undefined,
-    _workspaceId: string | undefined,
+    workspaceId: string | undefined,
     options: any,
   ) => {
     if (!workspaceConfigPath || typeof runner.acquireWorkspaceRuntime !== "function") return options;
+    const effectiveRelationships = workspaceId && d.effectiveRelationships
+      ? await d.effectiveRelationships.render(workspaceId)
+      : undefined;
     return {
       ...options,
-      runtimeConfig: await runner.acquireWorkspaceRuntime(workspaceConfigPath),
+      runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath, effectiveRelationships),
     };
   };
 
@@ -385,6 +358,7 @@ export function sessionRoutes(
       let workspaceId: string | undefined;
       let workspaceRevision: string | undefined;
       let workspaceDescriptor: WorkspaceDescriptor | undefined;
+      let allowedModels: readonly string[] | undefined;
       if (requestedWorkspaceId) {
         try {
           const registry = d.workspaceRegistry as Partial<WorkspaceRegistry>;
@@ -404,19 +378,7 @@ export function sessionRoutes(
           workspaceId = resolved.revision.id;
           workspaceRevision = resolved.revision.commit;
           workspaceDescriptor = resolved.workspace;
-          if (!await catalogTransportSupportsSessionRuntime(workspaceId)) {
-            return reply.code(409).send({
-              error: "This workspace transport is not available to runtime sessions.",
-              code: "workspace_not_activatable",
-            });
-          }
-          const fingerprint = await currentInputFingerprint(runner, workspaceConfigPath);
-          if (!await preprocessingIsCurrent(workspaceId, fingerprint)) {
-            return reply.code(409).send({
-              error: "Run workspace preprocessing before starting the core.",
-              code: "preprocessing_required",
-            });
-          }
+          allowedModels = resolved.workspace.llm_policy.allowed;
         } catch {
           return reply.code(409).send({
             error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE,
@@ -424,17 +386,12 @@ export function sessionRoutes(
           });
         }
       }
-	  const requestedCanonical = b.provider && b.model ? `${b.provider}/${b.model}` : undefined;
-	  let selectedCanonical = requestedCanonical ?? d.modelCatalog.defaultSession;
-	  let modelWarning: string | undefined;
-	  if (selectedCanonical && d.modelCatalog.defaultSession && !d.modelCatalog.hasSession(selectedCanonical)) {
-	    selectedCanonical = d.modelCatalog.defaultSession;
-	    modelWarning = `Configured model ${requestedCanonical ?? "selection"} is unavailable; using ${selectedCanonical}.`;
-	  }
-	  const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : undefined;
-	  const provider = selected?.provider ?? b.provider;
-	  const model = selected?.model ?? b.model;
+      const provider = b.provider ?? s.provider;
+      const model = b.model ?? s.model;
       const thinking = b.thinking ?? s.thinking;
+      if (allowedModels && provider && model && !allowedModels.includes(`${provider}/${model}`)) {
+        return reply.code(400).send({ error: "Selected model is not allowed by this workspace." });
+      }
       // A persisted session is resumable without keeping Pi alive. New work replaces every
       // runtime owned by this principal, while runtimes belonging to other users remain intact.
       // Optional chaining preserves the deliberately narrow manager stubs used by route tests.
@@ -533,7 +490,7 @@ export function sessionRoutes(
         ),
         () => d.mgr.start(id, rt, runtimeOptions),
       );
-      return { id, ...(modelWarning ? { warning: modelWarning } : {}) };
+      return { id };
     } finally {
       if (revisionLease && !manifestPersisted) {
         await revisionLease.abort().catch((error: unknown) => {
@@ -641,10 +598,6 @@ export function sessionRoutes(
         provider?: string; model?: string; thinking?: string;
         workspace_id?: string; workspace_revision?: string;
       };
-	  const savedCanonical = saved.provider && saved.model ? `${saved.provider}/${saved.model}` : "";
-	  if (d.modelCatalog.defaultSession && (!savedCanonical || !d.modelCatalog.hasSession(savedCanonical))) {
-	    return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
-	  }
       let workspaceConfigPath: string;
       let workspaceDescriptor: WorkspaceDescriptor | undefined;
       try {
@@ -653,23 +606,6 @@ export function sessionRoutes(
         workspaceDescriptor = resolved.workspace;
       }
       catch { return unavailableWorkspaceReply(reply); }
-      if (d.catalogRepository && saved.workspace_id
-        && !await catalogTransportSupportsSessionRuntime(saved.workspace_id)) {
-        return reply.code(409).send({
-          error: "This workspace transport is not available to runtime sessions.",
-          code: "workspace_not_activatable",
-        });
-      }
-      const fingerprint = d.catalogRepository
-        ? await currentInputFingerprint(runner, workspaceConfigPath)
-        : undefined;
-      if (d.catalogRepository
-        && (!saved.workspace_id || !await preprocessingIsCurrent(saved.workspace_id, fingerprint))) {
-        return reply.code(409).send({
-          error: "Run workspace preprocessing before starting the core.",
-          code: "preprocessing_required",
-        });
-      }
       try { settings = await d.getSettings(principal); } catch { return storageFailure(reply); }
       // This check belongs inside the per-session lock: a preceding cold Resume may have
       // installed a running runtime while this request was waiting.

backend/src/routes/settings.ts

@@ -1,27 +1,17 @@
 import type { FastifyInstance } from "fastify";
 import type { AppConfig } from "../config.js";
 import type { Settings } from "../settings/settings-store.js";
-import { listWorkspaces } from "./meta.js";
+import { listWorkspaces, type ListModelsFn } from "./meta.js";
 import type { PrincipalContext } from "../auth/principal.js";
 import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
-import {
-  splitCanonicalModelId,
-  type RuntimeModelCatalog,
-} from "../models/runtime-model-catalog.js";
 
-/** Merge only workspace and runtime-thinking preferences; model defaults belong to modelCatalog. */
-export function effectiveSettings(
-  cfg: AppConfig,
-  stored: Settings,
-  modelCatalog?: RuntimeModelCatalog,
-): Settings {
+/** Merge stored settings over env/first-workspace defaults. */
+export function effectiveSettings(cfg: AppConfig, stored: Settings): Settings {
   const workspaces = listWorkspaces(cfg.harnessDir);
-  const selected = modelCatalog?.defaultSession
-    ? splitCanonicalModelId(modelCatalog.defaultSession)
-    : undefined;
   return {
     workspace: stored.workspace ?? workspaces[0]?.name,
-    ...(selected ?? {}),
+    provider: cfg.defaults.provider ?? stored.provider,
+    model: cfg.defaults.model ?? stored.model,
     thinking: cfg.defaults.thinking ?? stored.thinking,
   };
 }
@@ -29,7 +19,7 @@ export function effectiveSettings(
 export function settingsRoutes(
   app: FastifyInstance,
   deps: {
-    cfg: AppConfig;
+    cfg: AppConfig; listModels: ListModelsFn;
     getSettings: (principal: PrincipalContext) => Promise<Settings>;
   },
 ): void {
@@ -47,6 +37,22 @@ export function settingsRoutes(
     const principal = requirePermission(req, reply, "settings.manage");
     if (!isPrincipalContext(principal)) return principal;
     const b = (req.body ?? {}) as Settings;
+    if (b.model) {
+      let available: { provider: string; id: string }[] = [];
+      try {
+        available = await deps.listModels();
+      } catch {
+        available = [];
+      }
+      // Only validate when Pi gave us a non-empty list; otherwise allow (degraded).
+      if (available.length > 0 && !available.some(
+        (candidate) => candidate.provider === b.provider && candidate.id === b.model,
+      )) {
+        return reply.code(400).send({
+          error: `Unknown model: ${b.provider ?? "unknown"}/${b.model}`,
+        });
+      }
+    }
     try {
       // Retain this endpoint as a validating compatibility surface for older clients, but do
       // not write anonymous users' choices to shared server storage.

backend/src/routes/workspace-preprocessing.ts

@@ -1,402 +0,0 @@
-import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
-import { z } from "zod";
-import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
-import {
-  CatalogUnavailableError,
-  type CatalogRepository,
-  type WorkspaceDatabase,
-} from "../catalog/types.js";
-import type { ThtRunner } from "../tht/tht-runner.js";
-import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
-import type { PreprocessingJobState } from "../workspaces/preprocessing-state.js";
-import type { WorkspaceRegistry } from "../workspaces/registry.js";
-
-const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
-const ACTIVE_SYNC_STATES = new Set(["queued", "running", "awaiting_confirmation", "applying"]);
-
-export type WorkspacePreprocessingUiState =
-  | "ready"
-  | "required"
-  | "running"
-  | "blocked"
-  | "failed";
-
-export interface WorkspacePreprocessingStatus {
-  schemaVersion: 1;
-  workspaceId: string;
-  state: WorkspacePreprocessingUiState;
-  actionable: boolean;
-  clearable: boolean;
-  detail: string;
-  reason?: string;
-  nextStep?: string;
-  metadataRevision?: number;
-  preprocessedMetadataRevision?: number;
-  startedAt?: string;
-  finishedAt?: string;
-  progress?: {
-    stage: "catalog_snapshot" | "schema_index" | "evidence" | "finalizing";
-    step: number;
-    totalSteps: 4;
-  };
-  lastFailure?: {
-    stage: string;
-    errorCode: string;
-    finishedAt: string;
-  };
-}
-
-export interface WorkspacePreprocessingRouteDeps {
-  repository: CatalogRepository;
-  registry: WorkspaceRegistry;
-  service: Pick<WorkspacePreprocessingService, "run" | "clear">;
-  inputFingerprint?: Pick<ThtRunner, "workspaceInputFingerprint">;
-  readLatestJob?: (workspaceId: string) => PreprocessingJobState | undefined;
-}
-
-const PROGRESS_DETAILS = {
-  catalog_snapshot: "Preparing the PostgreSQL Catalog snapshot.",
-  schema_index: "Building schema vectors and LSH indexes.",
-  evidence: "Indexing Evidence.",
-  finalizing: "Publishing the completed preprocessing state.",
-} as const;
-
-function runningProgress(
-  workspaceId: string,
-  deps: WorkspacePreprocessingRouteDeps,
-): NonNullable<WorkspacePreprocessingStatus["progress"]> {
-  let job: PreprocessingJobState | undefined;
-  try {
-    job = deps.readLatestJob?.(workspaceId);
-  } catch {
-    // Progress is supplemental. A damaged or temporarily unavailable checkpoint must not hide
-    // the authoritative running state held by PostgreSQL.
-  }
-  const completed = new Set(job?.status === "active" ? job.completedStages : []);
-  if (completed.has("evidence")) return { stage: "finalizing", step: 4, totalSteps: 4 };
-  if (completed.has("schema_index")) return { stage: "evidence", step: 3, totalSteps: 4 };
-  if (completed.has("catalog_snapshot")) return { stage: "schema_index", step: 2, totalSteps: 4 };
-  return { stage: "catalog_snapshot", step: 1, totalSteps: 4 };
-}
-
-function base(
-  workspaceId: string,
-  state: WorkspacePreprocessingUiState,
-  detail: string,
-  database?: WorkspaceDatabase,
-): WorkspacePreprocessingStatus {
-  return {
-    schemaVersion: 1,
-    workspaceId,
-    state,
-    actionable: state === "ready" || state === "required" || state === "failed",
-    clearable: Boolean(
-      database
-      && state !== "running"
-      && database.preprocessingErrorCode !== "derived_data_cleared"
-    ),
-    detail,
-    ...(database ? {
-      metadataRevision: database.metadataContentRevision,
-      ...(database.preprocessedMetadataRevision === undefined
-        ? {}
-        : { preprocessedMetadataRevision: database.preprocessedMetadataRevision }),
-      ...(database.preprocessingStartedAt ? { startedAt: database.preprocessingStartedAt } : {}),
-      ...(database.preprocessingFinishedAt ? { finishedAt: database.preprocessingFinishedAt } : {}),
-    } : {}),
-  };
-}
-
-function blocked(
-  workspaceId: string,
-  detail: string,
-  reason: string,
-  nextStep: string,
-  database?: WorkspaceDatabase,
-): WorkspacePreprocessingStatus {
-  return { ...base(workspaceId, "blocked", detail, database), reason, nextStep };
-}
-
-function failureDiagnostic(errorCode: string): {
-  stage: string;
-  detail: string;
-  reason: string;
-  nextStep: string;
-} {
-  switch (errorCode) {
-    case "catalog_snapshot_failed":
-      return {
-        stage: "catalog_snapshot",
-        detail: "The Catalog snapshot could not be prepared.",
-        reason: "PostgreSQL Catalog metadata could not be read into a consistent preprocessing snapshot.",
-        nextStep: "Check Catalog availability, then retry preprocessing.",
-      };
-    case "schema_index_failed":
-      return {
-        stage: "schema_index",
-        detail: "The schema index could not be rebuilt.",
-        reason: "The schema indexing worker stopped before the Catalog snapshot was published to Qdrant.",
-        nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
-      };
-    case "evidence_preprocessing_failed":
-      return {
-        stage: "evidence",
-        detail: "Evidence preprocessing did not complete.",
-        reason: "The Evidence indexing worker stopped before it finished publishing the current workspace data.",
-        nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
-      };
-    case "semantic_index_incompatible":
-      return {
-        stage: "semantic_preflight",
-        detail: "The semantic index configuration is incompatible.",
-        reason: "Qdrant rejected the collection configuration for the active embedding model.",
-        nextStep: "Check embedding dimensions and Qdrant collection settings, then retry.",
-      };
-    case "egress_policy_refused":
-      return {
-        stage: "evidence",
-        detail: "The Evidence source was refused by policy.",
-        reason: "The configured Evidence endpoint is not allowed by the installation egress policy.",
-        nextStep: "Correct the Evidence source or its allowlist configuration, then retry.",
-      };
-    case "workspace_not_activatable":
-      return {
-        stage: "runtime_preflight",
-        detail: "The workspace runtime could not be prepared.",
-        reason: "The active workspace or one of its required runtime bindings is not usable.",
-        nextStep: "Check Workspace management and Database management, then retry.",
-      };
-    default:
-      return {
-        stage: "preprocessing",
-        detail: "Preprocessing did not complete.",
-        reason: "The preprocessing worker stopped before the current Catalog revision was published.",
-        nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
-      };
-  }
-}
-
-export async function readWorkspacePreprocessingStatus(
-  workspaceId: string,
-  deps: WorkspacePreprocessingRouteDeps,
-): Promise<WorkspacePreprocessingStatus> {
-  const database = await deps.repository.getByWorkspace(workspaceId);
-  if (!database) {
-    return blocked(
-      workspaceId,
-      "Database configuration is required.",
-      "This workspace has no database configuration in the PostgreSQL Catalog.",
-      "Open Database management and configure the workspace database.",
-    );
-  }
-
-  if (database.preprocessingStatus === "running") {
-    const progress = runningProgress(workspaceId, deps);
-    return {
-      ...base(workspaceId, "running", PROGRESS_DETAILS[progress.stage], database),
-      progress,
-    };
-  }
-  if (database.binding.transport === "ssh_tunnel") {
-    return blocked(
-      workspaceId,
-      "The database transport is not supported by the core runtime.",
-      "The current database binding uses an SSH tunnel, which cannot be used by a ThothII session.",
-      "Open Database management and select a supported runtime transport.",
-      database,
-    );
-  }
-  if (database.schemaSyncedVersion !== database.version) {
-    return blocked(
-      workspaceId,
-      "Catalog synchronization is required.",
-      `Database configuration v${database.version} is newer than the latest Catalog synchronization${database.schemaSyncedVersion === undefined ? "." : ` v${database.schemaSyncedVersion}.`}`,
-      "Open Database management and run Synchronize schema.",
-      database,
-    );
-  }
-
-  const [syncRuns, activeDescriptionRun, sensitivityRuns] = await Promise.all([
-    deps.repository.listSyncRuns(database.id, 10),
-    deps.repository.getActiveDescriptionGenerationRun(),
-    deps.repository.listSensitivityAnalysisRuns(50),
-  ]);
-  if (syncRuns.some((run) => ACTIVE_SYNC_STATES.has(run.state))) {
-    return blocked(
-      workspaceId,
-      "Catalog synchronization is in progress.",
-      "The Catalog is being synchronized and its metadata revision is not stable yet.",
-      "Wait for schema synchronization to finish, then run preprocessing.",
-      database,
-    );
-  }
-  if (activeDescriptionRun?.databaseId === database.id
-    && ["queued", "running"].includes(activeDescriptionRun.status)) {
-    return blocked(
-      workspaceId,
-      "Description generation is in progress.",
-      "Catalog descriptions are still being generated for this database.",
-      "Wait for description generation to finish, then run preprocessing.",
-      database,
-    );
-  }
-  if (sensitivityRuns.some((run) => run.databaseId === database.id && run.status === "running")) {
-    return blocked(
-      workspaceId,
-      "Sensitivity analysis is in progress.",
-      "Catalog sensitivity metadata is still being analyzed for this database.",
-      "Wait for sensitivity analysis to finish, then run preprocessing.",
-      database,
-    );
-  }
-
-  let inputFingerprint: string | undefined;
-  try {
-    const record = await deps.registry.read(workspaceId);
-    if (deps.inputFingerprint) {
-      inputFingerprint = await deps.inputFingerprint.workspaceInputFingerprint(
-        record.revision.snapshotPath,
-      );
-    }
-  } catch {
-    return blocked(
-      workspaceId,
-      "The workspace runtime configuration is unavailable.",
-      "The active workspace revision or one of its required database secrets could not be resolved.",
-      "Check Workspace management and Database management before running preprocessing.",
-      database,
-    );
-  }
-
-  const current = database.preprocessingStatus === "succeeded"
-    && database.preprocessedMetadataRevision === database.metadataContentRevision
-    && (inputFingerprint === undefined
-      || database.preprocessingInputFingerprint === inputFingerprint);
-  if (current) {
-    return base(
-      workspaceId,
-      "ready",
-      `Catalog revision ${database.metadataContentRevision} is indexed.`,
-      database,
-    );
-  }
-
-  if (database.preprocessingErrorCode === "derived_data_cleared") {
-    return base(
-      workspaceId,
-      "required",
-      "Reference vectors and LSH are empty. Memory is preserved.",
-      database,
-    );
-  }
-
-  if (database.preprocessingStatus === "failed"
-    && database.preprocessingErrorCode
-    && database.preprocessingErrorCode !== "catalog_changed"
-    && database.preprocessingErrorCode !== "derived_data_cleared"
-    && database.preprocessingFinishedAt) {
-    const diagnostic = failureDiagnostic(database.preprocessingErrorCode);
-    return {
-      ...base(workspaceId, "failed", diagnostic.detail, database),
-      reason: diagnostic.reason,
-      nextStep: diagnostic.nextStep,
-      lastFailure: {
-        stage: diagnostic.stage,
-        errorCode: database.preprocessingErrorCode,
-        finishedAt: database.preprocessingFinishedAt,
-      },
-    };
-  }
-
-  return base(
-    workspaceId,
-    "required",
-    `Catalog revision ${database.metadataContentRevision} is not indexed.`,
-    database,
-  );
-}
-
-function safeError(reply: FastifyReply, error: unknown) {
-  if (error instanceof CatalogUnavailableError) {
-    return reply.code(503).send({
-      code: "catalog_unavailable",
-      message: "Database catalog is unavailable.",
-    });
-  }
-  if (error instanceof z.ZodError) {
-    return reply.code(400).send({
-      code: "preprocessing_request_invalid",
-      message: "Preprocessing request is invalid.",
-    });
-  }
-  return reply.code(500).send({
-    code: "preprocessing_run_failed",
-    message: "Preprocessing status could not be resolved.",
-  });
-}
-
-function workspaceIdFrom(request: FastifyRequest): string {
-  return workspaceIdSchema.parse((request.params as { workspaceId?: unknown }).workspaceId);
-}
-
-export function workspacePreprocessingRoutes(
-  app: FastifyInstance,
-  deps: WorkspacePreprocessingRouteDeps,
-): void {
-  app.get("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
-    if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply;
-    try {
-      return await readWorkspacePreprocessingStatus(workspaceIdFrom(request), deps);
-    } catch (error) {
-      return safeError(reply, error);
-    }
-  });
-
-  app.post("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
-    if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
-    try {
-      const workspaceId = workspaceIdFrom(request);
-      const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
-      if (!before.actionable) {
-        return reply.code(409).send({ ...before, code: "preprocessing_blocked" });
-      }
-
-      const result = await deps.service.run({ workspaceId });
-      const after = await readWorkspacePreprocessingStatus(workspaceId, deps);
-      if (after.state === "ready") return after;
-      if (after.state === "failed") {
-        return reply.code(422).send({ ...after, code: "preprocessing_run_failed" });
-      }
-      if (after.state === "blocked" || after.state === "running") {
-        return reply.code(409).send({ ...after, code: "preprocessing_blocked" });
-      }
-      return reply.code(500).send({
-        code: "preprocessing_run_failed",
-        message: `Preprocessing ended with ${result.code}.`,
-      });
-    } catch (error) {
-      return safeError(reply, error);
-    }
-  });
-
-  app.delete("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
-    if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
-    try {
-      const workspaceId = workspaceIdFrom(request);
-      const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
-      if (!before.clearable) {
-        return reply.code(409).send({ ...before, code: "preprocessing_clear_blocked" });
-      }
-      const result = await deps.service.clear({ workspaceId });
-      if (result.status !== "succeeded") {
-        return reply.code(result.code === "preprocessing_conflict" ? 409 : 500).send({
-          code: result.code,
-          message: "Preprocessing data could not be cleared.",
-        });
-      }
-      return await readWorkspacePreprocessingStatus(workspaceId, deps);
-    } catch (error) {
-      return safeError(reply, error);
-    }
-  });
-}

backend/src/routes/workspaces.ts

@@ -16,33 +16,23 @@ import {
   type WorkspaceDescriptor,
 } from "../workspaces/schema.js";
 import type { RuntimeBindings } from "../workspaces/runtime-renderer.js";
-import type {
-  ConnectorDiagnostics,
-  Diagnostic,
-  WorkspaceDiagnosticOptions,
-} from "../workspaces/diagnostics.js";
+import type { ConnectorDiagnostics } from "../workspaces/diagnostics.js";
 import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
 import type { AuthDiagnoser } from "../auth/diagnostics.js";
 import { decodeAuthDiagnostics, type AuthDiagnostics } from "../auth/group-catalog.js";
-import type { WorkspaceDatabase } from "../catalog/types.js";
 
 export type WorkspaceDiagnoser = (
   workspace: WorkspaceDescriptor,
   bindings: RuntimeBindings,
-  options: WorkspaceDiagnosticOptions,
+  options: { writeProbe: boolean },
 ) => Promise<ConnectorDiagnostics>;
 
-export type WorkspaceDatabaseTester = (
-  workspaceId: string,
-) => Promise<WorkspaceDatabase | undefined>;
-
 interface WorkspaceRoutesDeps {
   registry: WorkspaceRegistry;
   config: WorkspaceRegistryConfig;
   diagnose: WorkspaceDiagnoser;
   authDiagnoser: AuthDiagnoser;
   secretStore: WorkspaceSecretStore;
-  testDatabaseConnection: WorkspaceDatabaseTester;
 }
 
 const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
@@ -66,20 +56,6 @@ const SAFE_MESSAGES = {
   semantic_index_incompatible: "Semantic index is incompatible with this workspace.",
 } as const;
 
-const catalogConnectionUnavailable = (): Diagnostic => ({
-  level: "error",
-  code: "connector_unavailable",
-  field: "dwh",
-  message: "The configured database could not be reached or authenticated.",
-});
-
-const catalogConnectionMissing = (): Diagnostic => ({
-  level: "error",
-  code: "binding_missing",
-  field: "dwh",
-  message: "Configure this workspace in Database Management before testing connections.",
-});
-
 function authenticationReport(value: unknown): AuthDiagnostics {
   const report = decodeAuthDiagnostics(value);
   if (!report) throw new Error("invalid authentication diagnostic report");
@@ -262,33 +238,14 @@ export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps)
         deps.secretStore,
       );
       try {
-        const [workspaceDiagnostics, testedDatabase, inspectedAuthentication] = await Promise.all([
-          deps.diagnose(operational, lease.bindings, {
-            writeProbe: false,
-            skipDwh: true,
-          }),
-          deps.testDatabaseConnection(id),
+        const [workspaceDiagnostics, inspectedAuthentication] = await Promise.all([
+          deps.diagnose(operational, lease.bindings, { writeProbe: false }),
           deps.authDiagnoser.inspect({ live: true }),
         ]);
         const authentication = authenticationReport(inspectedAuthentication);
-        const catalogConnectionReady = testedDatabase?.connectionStatus === "reachable";
-        const catalogConnectionDiagnostic = !testedDatabase
-          ? catalogConnectionMissing()
-          : catalogConnectionReady
-            ? undefined
-            : catalogConnectionUnavailable();
-        const diagnostics = catalogConnectionDiagnostic
-          ? [
-            ...workspaceDiagnostics.diagnostics.filter(({ code }) => code !== "binding_ok"),
-            catalogConnectionDiagnostic,
-          ]
-          : workspaceDiagnostics.diagnostics;
         return {
           ...workspaceDiagnostics,
-          activatable: workspaceDiagnostics.activatable
-            && catalogConnectionReady
-            && authentication.ready,
-          diagnostics,
+          activatable: workspaceDiagnostics.activatable && authentication.ready,
           authentication,
         };
       } finally {

backend/src/settings/settings-store.ts

@@ -13,7 +13,6 @@ import type { AppConfig } from "../config.js";
 
 export interface Settings {
   workspace?: string;
-  /** Legacy input fields are ignored when loading/evaluating installation settings. */
   provider?: string;
   model?: string;
   thinking?: string;
@@ -38,13 +37,7 @@ export function loadSettings(cfg: AppConfig): Settings {
   try {
     const raw = readFileSync(cfg.settingsFile, "utf8");
     const parsed = JSON.parse(raw);
-    if (parsed && typeof parsed === "object") {
-      const value = parsed as Record<string, unknown>;
-      return {
-        ...(typeof value.workspace === "string" ? { workspace: value.workspace } : {}),
-        ...(typeof value.thinking === "string" ? { thinking: value.thinking } : {}),
-      };
-    }
+    if (parsed && typeof parsed === "object") return parsed as Settings;
     return {};
   } catch {
     return {};

backend/src/tht/tht-runner.ts

@@ -5,7 +5,7 @@ import {
   openSync, readFileSync, readSync, realpathSync, statSync, unlinkSync, writeFileSync,
 } from "node:fs";
 import { dirname, isAbsolute, join, relative, resolve } from "node:path";
-import { parse, parseAllDocuments } from "yaml";
+import { parseAllDocuments } from "yaml";
 import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js";
 import { secretValue, type SecretBundleConfig } from "../config/secret-bundle.js";
 import { renderWorkspaceRuntimeFromSnapshotPath } from "../workspaces/runtime-config-lease.js";
@@ -22,9 +22,6 @@ import {
 } from "../workspaces/schema.js";
 import { reconcileCollection, type CollectionMode } from "../workspaces/qdrant-collection.js";
 import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
-import type { CatalogRepository } from "../catalog/types.js";
-import { preprocessingInputFingerprint } from "../workspaces/effective-config.js";
-import { workspaceVectorCollections } from "../workspaces/vector-collections.js";
 
 export interface ThtConfig extends SecretBundleConfig {
   thtBin: string;
@@ -38,14 +35,12 @@ export interface ThtConfig extends SecretBundleConfig {
   /** "self_heal" for session admission (create missing collections/indexes), default "require_existing". */
   qdrantCollectionMode?: "self_heal" | "require_existing";
   workspaceSecretStore?: WorkspaceSecretStore;
-  catalogRepository?: CatalogRepository;
 }
 
 export interface RuntimeConfigLease {
   path: string;
   workspaceId: string;
   workspaceRevision: string;
-  inputFingerprint: string;
   release(): void;
 }
 
@@ -84,7 +79,6 @@ export type SemanticReadinessCode = "workspace_not_activatable" | "semantic_inde
 export interface QdrantEnsureResult {
   ok: boolean;
   code?: SemanticReadinessCode;
-  state?: "ready" | "created" | "repaired" | "upgraded";
 }
 
 const REQUIRED_QDRANT_PAYLOAD_INDEXES = [
@@ -219,31 +213,37 @@ export class ThtRunner {
   }
 
   /** Render one immutable canonical registry revision into a backend-owned harness config. */
-  async acquireWorkspaceRuntime(workspaceConfigPath: string): Promise<RuntimeConfigLease> {
-    const identity = this.assertWorkspaceSnapshot(workspaceConfigPath);
-    const catalogDatabase = this.cfg.catalogRepository === undefined
+  acquireWorkspaceRuntime(
+    workspaceConfigPath: string,
+    effectiveRelationships?: string,
+  ): RuntimeConfigLease {
+    const effectiveRelationshipsPath = effectiveRelationships === undefined
       ? undefined
-      : await this.cfg.catalogRepository.getByWorkspace(identity.workspaceId);
-    if (this.cfg.catalogRepository !== undefined && !catalogDatabase) {
-      throw new Error("workspace database is not configured in the Catalog");
+      : this.createRuntimeSnapshot(effectiveRelationships);
+    let rendered: ReturnType<typeof renderWorkspaceRuntimeFromSnapshotPath>;
+    try {
+      rendered = renderWorkspaceRuntimeFromSnapshotPath({
+        snapshotPath: workspaceConfigPath,
+        harnessDir: this.cfg.harnessDir,
+        configPath: this.cfg.configPath,
+        dataRoot: this.cfg.dataRoot ?? (() => {
+          throw new Error("registry workspace runtime requires an absolute data root");
+        })(),
+        secretRoots: this.cfg.secretRoots ?? [],
+        semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
+        workspaceSecretStore: this.cfg.workspaceSecretStore,
+        effectiveRelationshipsPath,
+      });
+    } catch (error) {
+      if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
+      throw error;
     }
-    const rendered = renderWorkspaceRuntimeFromSnapshotPath({
-      snapshotPath: workspaceConfigPath,
-      harnessDir: this.cfg.harnessDir,
-      configPath: this.cfg.configPath,
-      dataRoot: this.cfg.dataRoot ?? (() => {
-        throw new Error("registry workspace runtime requires an absolute data root");
-      })(),
-      secretRoots: this.cfg.secretRoots ?? [],
-      semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
-      workspaceSecretStore: this.cfg.workspaceSecretStore,
-      catalogDatabase,
-    });
     let path: string;
     try {
       path = this.createRuntimeSnapshot(rendered.renderedConfig);
     } catch (error) {
       rendered.releaseSecrets();
+      if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
       throw error;
     }
     let released = false;
@@ -251,29 +251,16 @@ export class ThtRunner {
       path,
       workspaceId: rendered.workspaceId,
       workspaceRevision: rendered.workspaceRevision,
-      inputFingerprint: preprocessingInputFingerprint(
-        rendered.workspaceId,
-        rendered.workspaceRevision,
-        parse(rendered.renderedConfig),
-      ),
       release: () => {
         if (released) return;
         released = true;
         this.cleanupRuntimeSnapshot(path);
+        if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
         rendered.releaseSecrets();
       },
     };
   }
 
-  async workspaceInputFingerprint(workspaceConfigPath: string): Promise<string> {
-    const lease = await this.acquireWorkspaceRuntime(workspaceConfigPath);
-    try {
-      return lease.inputFingerprint;
-    } finally {
-      lease.release();
-    }
-  }
-
   private runtimeSnapshotDirectory(): string {
     if (!this.cfg.runtimeSnapshotRoot) throw new Error("runtime snapshot root is not configured");
     if (!isAbsolute(this.cfg.runtimeSnapshotRoot)) throw new Error("runtime snapshot root must be absolute");
@@ -405,9 +392,13 @@ export class ThtRunner {
       workspaceConfigPath && isAbsolute(workspaceConfigPath)
       && !this.runtimeSnapshots.has(workspaceConfigPath)
     ) {
-      return this.acquireWorkspaceRuntime(workspaceConfigPath).then((runtime) => (
-        this.run(args, runtime.path, timeoutMs).finally(runtime.release)
-      ));
+      let runtime: RuntimeConfigLease;
+      try {
+        runtime = this.acquireWorkspaceRuntime(workspaceConfigPath);
+      } catch (error) {
+        return Promise.reject(error);
+      }
+      return this.run(args, runtime.path, timeoutMs).finally(runtime.release);
     }
     return new Promise((resolve) => {
       const env: NodeJS.ProcessEnv = { ...process.env };
@@ -607,26 +598,20 @@ export class ThtRunner {
     } catch {
       return { ok: false, code: "workspace_not_activatable" };
     }
-    const collections = workspaceVectorCollections(descriptor.workspace.id);
+    const collection = descriptor.semantic_index.vector_store;
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutSec) * 1000);
     try {
-      const checked = await Promise.all(Object.entries(collections).map(async ([purpose, collection]) =>
-        await reconcileCollection({
-          baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
-          collection,
-          dimensions: this.cfg.semanticRuntime.internalEmbeddingDimensions,
-          distance: "cosine",
-          mode: mode === "evidence_maintenance" && purpose === "memory" ? "self_heal" : mode,
-          request: this.cfg.qdrantRequest ?? fetch,
-          signal: controller.signal,
-        })));
-      if (!checked.every((result) => result.ok)) {
-        return { ok: false, code: "semantic_index_incompatible" };
-      }
-      const state = (["upgraded", "repaired", "created", "ready"] as const)
-        .find((candidate) => checked.some((result) => result.state === candidate));
-      return { ok: true, ...(state ? { state } : {}) };
+      const checked = await reconcileCollection({
+        baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
+        collection: collection.collection,
+        dimensions: collection.dimensions,
+        distance: collection.distance,
+        mode,
+        request: this.cfg.qdrantRequest ?? fetch,
+        signal: controller.signal,
+      });
+      return checked;
     } catch {
       return { ok: false, code: "workspace_not_activatable" };
     } finally {

backend/src/workspace-maintenance.ts

@@ -1,14 +1,15 @@
 import { spawn } from "node:child_process";
 import { closeSync, constants as fsConstants, openSync } from "node:fs";
+import { readdir, readFile } from "node:fs/promises";
 import { join } from "node:path";
-import { loadConfig, type AppConfig } from "./config.js";
+import { parse } from "yaml";
+import { loadConfig } from "./config.js";
 import { ThtRunner } from "./tht/tht-runner.js";
 import { WorkspaceRegistry } from "./workspaces/registry.js";
 import { publishDeterministicRuntimeConfigLease, renderActiveWorkspaceRuntime } from "./workspaces/runtime-config-lease.js";
 import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
 import { WorkspacePreprocessingService, type WorkspaceOperationResult } from "./workspaces/preprocessing-service.js";
-import { createCatalogRepository } from "./catalog/repository.js";
-import type { CatalogRepository } from "./catalog/types.js";
+import type { SessionInventoryRow } from "./workspaces/preprocessing-state.js";
 
 export interface WorkspaceMaintenanceIo {
   stdin: string;
@@ -18,7 +19,7 @@ export interface WorkspaceMaintenanceIo {
   writeStderr(value: string): void;
 }
 
-type Command = "inspect" | "preprocess-run" | "preprocess-clear";
+type Command = "inspect" | "preprocess-dwh" | "schema-suggest-fks" | "schema-check" | "schema-accept" | "index-schema" | "preprocess-evidence" | "preprocess-run" | "vector-inspect" | "vector-rebuild";
 
 function failureResult(
   operation: string,
@@ -63,8 +64,15 @@ function parseRequest(command: string, stdin: string): Record<string, unknown> {
   }
   const allowedByCommand: Record<string, readonly string[]> = {
     inspect: ["schemaVersion", "workspaceId"],
-    "preprocess-run": ["schemaVersion", "workspaceId"],
-    "preprocess-clear": ["schemaVersion", "workspaceId"],
+    "preprocess-dwh": ["schemaVersion", "workspaceId", "resumeRunId"],
+    "schema-suggest-fks": ["schemaVersion", "workspaceId", "fromSql", "assume", "resumeRunId"],
+    "schema-check": ["schemaVersion", "workspaceId", "annotationsYaml", "reviewedCandidatesDigest"],
+    "schema-accept": ["schemaVersion", "workspaceId", "runId", "yes"],
+    "index-schema": ["schemaVersion", "workspaceId", "resumeRunId"],
+    "preprocess-evidence": ["schemaVersion", "workspaceId", "dryRun", "resumeRunId"],
+    "preprocess-run": ["schemaVersion", "workspaceId", "resumeRunId"],
+    "vector-inspect": ["schemaVersion", "workspaceId"],
+    "vector-rebuild": ["schemaVersion", "workspaceId", "collection", "confirm", "destroy"],
   };
   const allowed = allowedByCommand[command];
   if (!allowed) throw new Error("unknown command");
@@ -83,12 +91,55 @@ async function dispatch(command: Command, service: WorkspacePreprocessingService
   switch (command) {
     case "inspect":
       return await service.inspect({ workspaceId: request.workspaceId as string });
+    case "preprocess-dwh":
+      return await service.preprocessDwh({
+        workspaceId: request.workspaceId as string,
+        resumeRunId: request.resumeRunId as string | undefined,
+      });
+    case "schema-suggest-fks":
+      return await service.suggestFks({
+        workspaceId: request.workspaceId as string,
+        fromSql: request.fromSql as any,
+        assume: request.assume as any,
+        resumeRunId: request.resumeRunId as string | undefined,
+      });
+    case "schema-check":
+      return await service.checkSchema({
+        workspaceId: request.workspaceId as string,
+        annotationsYaml: request.annotationsYaml as string | undefined,
+        reviewedCandidatesDigest: request.reviewedCandidatesDigest as string | undefined,
+      });
+    case "schema-accept":
+      return await service.acceptSchema({
+        workspaceId: request.workspaceId as string,
+        runId: request.runId as string,
+        yes: request.yes === true,
+      });
+    case "index-schema":
+      return await service.indexSchema({
+        workspaceId: request.workspaceId as string,
+        resumeRunId: request.resumeRunId as string | undefined,
+      });
+    case "preprocess-evidence":
+      return await service.preprocessEvidence({
+        workspaceId: request.workspaceId as string,
+        dryRun: request.dryRun as boolean | undefined,
+        resumeRunId: request.resumeRunId as string | undefined,
+      });
     case "preprocess-run":
       return await service.run({
         workspaceId: request.workspaceId as string,
+        resumeRunId: request.resumeRunId as string | undefined,
+      });
+    case "vector-inspect":
+      return await service.vectorInspect({ workspaceId: request.workspaceId as string });
+    case "vector-rebuild":
+      return await service.vectorRebuild({
+        workspaceId: request.workspaceId as string,
+        collection: request.collection as string | undefined,
+        confirm: request.confirm as string | undefined,
+        destroy: request.destroy === true,
       });
-    case "preprocess-clear":
-      return await service.clear({ workspaceId: request.workspaceId as string });
   }
 }
 
@@ -127,31 +178,48 @@ export async function runWorkspaceMaintenanceCli(
       || message === "unexpected request field"
       || message === "invalid workspace id";
     return command in {
-      inspect: true, "preprocess-run": true, "preprocess-clear": true,
+      inspect: true, "preprocess-dwh": true, "schema-suggest-fks": true, "schema-check": true,
+      "schema-accept": true,
+      "index-schema": true, "preprocess-evidence": true, "preprocess-run": true,
     } ? (requestError ? 2 : 1) : 2;
   }
 }
 
-export interface ProductionWorkspacePreprocessingDeps {
-  config?: AppConfig;
-  catalogRepository?: CatalogRepository;
-  registry?: WorkspaceRegistry;
-  workspaceSecretStore?: WorkspaceSecretStore;
-  runner?: ThtRunner;
+async function readSessionInventory(dataRoot: string, workspaceId: string): Promise<readonly SessionInventoryRow[]> {
+  const directory = join(dataRoot, "sessions", workspaceId, "sessions");
+  try {
+    const entries = await readdir(directory, { withFileTypes: true });
+    const rows: SessionInventoryRow[] = [];
+    for (const entry of entries) {
+      if (!entry.isDirectory() || entry.isSymbolicLink()) continue;
+      try {
+        const source = await readFile(join(directory, entry.name, "session_manifest.yaml"), "utf8");
+        const manifest = parse(source) as Record<string, unknown>;
+        rows.push({
+          id: entry.name,
+          status: typeof manifest.status === "string" ? manifest.status : "open",
+          archived: manifest.archived === true,
+          workspaceRevision: typeof manifest.workspace_revision === "string" ? manifest.workspace_revision : null,
+        });
+      } catch {
+        // fail closed at mutation time by ignoring unreadable manifests from the resumable scan
+      }
+    }
+    return rows;
+  } catch {
+    return [];
+  }
 }
 
-export function createProductionWorkspacePreprocessingService(
-  deps: ProductionWorkspacePreprocessingDeps = {},
-): WorkspacePreprocessingService {
-  const config = deps.config ?? loadConfig(process.env, { surface: "workspace-maintenance" });
-  const catalogRepository = deps.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
-  const registry = deps.registry ?? new WorkspaceRegistry(config.workspaceRegistry);
-  const workspaceSecretStore = deps.workspaceSecretStore ?? new WorkspaceSecretStore({
+function createProductionService(): WorkspacePreprocessingService {
+  const config = loadConfig(process.env, { surface: "workspace-maintenance" });
+  const registry = new WorkspaceRegistry(config.workspaceRegistry);
+  const workspaceSecretStore = new WorkspaceSecretStore({
     root: config.workspaceSecretStoreRoot,
     runtimeRoot: config.workspaceSecretRuntimeRoot,
     installationId: config.workspaceRegistry.installationId,
   });
-  const runner = deps.runner ?? new ThtRunner({
+  const runner = new ThtRunner({
     thtBin: config.thtBin,
     harnessDir: config.harnessDir,
     configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
@@ -164,7 +232,6 @@ export function createProductionWorkspacePreprocessingService(
     semanticRuntime: {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
-      internalEmbeddingId: config.internalEmbeddingId,
       internalEmbeddingModel: config.internalEmbeddingModel,
       internalEmbeddingDimensions: config.internalEmbeddingDimensions,
     },
@@ -173,10 +240,7 @@ export function createProductionWorkspacePreprocessingService(
     dataRoot: config.dataRoot ?? "/data",
     httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "")
       .split(",").map((value) => value.trim()).filter((value) => value.length > 0),
-    catalogRepository,
     acquireActiveRuntime: async (workspaceId) => {
-      const catalogDatabase = await catalogRepository.getByWorkspace(workspaceId);
-      if (!catalogDatabase) throw new Error("workspace database is not configured in the Catalog");
       const active = await renderActiveWorkspaceRuntime({
         workspaceId,
         registry,
@@ -186,7 +250,6 @@ export function createProductionWorkspacePreprocessingService(
         dataRoot: config.dataRoot ?? "/data",
         secretRoots: config.workspaceRegistry.secretRoots,
         workspaceSecretStore,
-        catalogDatabase,
         semanticRuntime: {
           internalQdrantUrl: config.internalQdrantUrl,
           internalEmbeddingUrl: config.internalEmbeddingUrl,
@@ -194,55 +257,38 @@ export function createProductionWorkspacePreprocessingService(
           internalEmbeddingDimensions: config.internalEmbeddingDimensions,
         },
       });
-      try {
-        const configLease = await publishDeterministicRuntimeConfigLease({
-          workspaceId,
-          registry,
-          registryConfig: config.workspaceRegistry,
-          harnessDir: config.harnessDir,
-          configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
-          dataRoot: config.dataRoot ?? "/data",
-          secretRoots: config.workspaceRegistry.secretRoots,
-          semanticRuntime: {
-            internalQdrantUrl: config.internalQdrantUrl,
-            internalEmbeddingUrl: config.internalEmbeddingUrl,
-            internalEmbeddingModel: config.internalEmbeddingModel,
-            internalEmbeddingDimensions: config.internalEmbeddingDimensions,
-          },
-          workspaceSecretStore,
-          catalogDatabase,
-        });
-        return {
-          workspace: active.workspace,
-          workspaceId: active.workspaceId,
-          workspaceRevision: active.workspaceRevision,
-          descriptorBlob: active.descriptorBlob,
-          catalogBlob: active.catalogBlob,
-          configLease,
-        };
-      } finally {
-        active.releaseSecrets();
-      }
+      const configLease = await publishDeterministicRuntimeConfigLease({
+        workspaceId,
+        registry,
+        registryConfig: config.workspaceRegistry,
+        harnessDir: config.harnessDir,
+        configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
+        dataRoot: config.dataRoot ?? "/data",
+        secretRoots: config.workspaceRegistry.secretRoots,
+        semanticRuntime: {
+          internalQdrantUrl: config.internalQdrantUrl,
+          internalEmbeddingUrl: config.internalEmbeddingUrl,
+          internalEmbeddingModel: config.internalEmbeddingModel,
+          internalEmbeddingDimensions: config.internalEmbeddingDimensions,
+        },
+        workspaceSecretStore,
+      });
+      return {
+        workspace: active.workspace,
+        workspaceId: active.workspaceId,
+        workspaceRevision: active.workspaceRevision,
+        descriptorBlob: active.descriptorBlob,
+        catalogBlob: active.catalogBlob,
+        configLease,
+      };
     },
     runChild: async ({ argv, configPath }) => {
       const configFd = openSync(configPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
       try {
         return await new Promise((resolve) => {
-          const childEnvironment = { ...process.env };
-          for (const name of [
-            "THT_CATALOG_DATABASE_URL",
-            "THT_CATALOG_DB_HOST",
-            "THT_CATALOG_DB_PORT",
-            "THT_CATALOG_DB_NAME",
-            "THT_CATALOG_RUNTIME_USER",
-            "THT_CATALOG_RUNTIME_PASSWORD_FILE",
-            "THT_CATALOG_MIGRATOR_DATABASE_URL",
-            "THT_CATALOG_MIGRATOR_USER",
-            "THT_CATALOG_MIGRATOR_PASSWORD_FILE",
-          ]) delete childEnvironment[name];
           const child = spawn(config.thtBin, argv, {
             cwd: config.harnessDir,
-            env: { ...childEnvironment, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
+            env: { ...process.env, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
             stdio: ["ignore", "pipe", "pipe", configFd],
           });
           let stdout = "";
@@ -256,8 +302,9 @@ export function createProductionWorkspacePreprocessingService(
         closeSync(configFd);
       }
     },
+    listSessions: async (workspaceId) => await readSessionInventory(config.dataRoot ?? "/data", workspaceId),
     semanticPreflight: async (workspace) => {
-      const result = await runner.qdrantEnsure(workspace, 30, "self_heal");
+      const result = await runner.qdrantEnsure(workspace, 30);
       return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" };
     },
     evidencePreflight: async (workspace) => {
@@ -282,11 +329,7 @@ if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).
     writeStdout: (value) => { stdout.push(value); },
     writeStderr: (value) => { stderr.push(value); },
   };
-  const exitCode = await runWorkspaceMaintenanceCli(
-    process.argv,
-    createProductionWorkspacePreprocessingService(),
-    io,
-  );
+  const exitCode = await runWorkspaceMaintenanceCli(process.argv, createProductionService(), io);
   process.stdout.write(stdout.join(""));
   if (stderr.length > 0) process.stderr.write(stderr.join("").slice(0, 64 * 1024));
   process.exit(exitCode);

backend/src/workspaces/bindings.ts

@@ -59,12 +59,12 @@ function safeSecretFilePath(path: string, secretRoots: readonly string[]): strin
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Workspace bindings support only workspace schema version 4");
+    throw new Error("Workspace bindings support only workspace schema version 3");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 4) {
-    throw new Error("Workspace bindings support only workspace schema version 4");
+    || Reflect.get(metadata, "schema_version") !== 3) {
+    throw new Error("Workspace bindings support only workspace schema version 3");
   }
 }
 
@@ -90,7 +90,6 @@ export function resolveBinding(
 ): ResolvedBinding {
   requireSupportedDescriptor(workspace);
   const descriptor = validateWorkspaceDescriptor(workspace);
-  if (!descriptor.dwh) throw new Error("workspace database is not bound in the descriptor");
   const contract = buildInstallationContract(descriptor);
   const variables = contract.variables.filter((variable) => variable.role === role);
   const transportVariable = variables.find((variable) => variable.suffix === "TRANSPORT");
@@ -156,7 +155,7 @@ export function resolveEvidenceBinding(
   return { values, missing };
 }
 
-/** Resolve the complete schema-v4 runtime binding set. */
+/** Resolve the complete schema-v3 runtime binding set. */
 export function resolveRuntimeBindings(
   workspace: WorkspaceDescriptor,
   env: NodeJS.ProcessEnv,
@@ -166,9 +165,7 @@ export function resolveRuntimeBindings(
   const descriptor = validateWorkspaceDescriptor(workspace);
 
   return {
-    dwh: descriptor.dwh
-      ? resolveBinding(descriptor, "DWH", env, secretRoots)
-      : { transport: "postgres_direct", values: {}, missing: [] },
+    dwh: resolveBinding(descriptor, "DWH", env, secretRoots),
     evidence: resolveEvidenceBinding(descriptor, env, secretRoots),
   };
 }

backend/src/workspaces/contracts.ts

@@ -137,12 +137,12 @@ function evidenceVariables(
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Installation contract supports only workspace schema version 4");
+    throw new Error("Installation contract supports only workspace schema version 3");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 4) {
-    throw new Error("Installation contract supports only workspace schema version 4");
+    || Reflect.get(metadata, "schema_version") !== 3) {
+    throw new Error("Installation contract supports only workspace schema version 3");
   }
 }
 
@@ -155,9 +155,7 @@ export function buildInstallationContract(workspace: WorkspaceDescriptor): Insta
     workspaceId: descriptor.workspace.id,
     namespace,
     variables: [
-      ...(descriptor.dwh
-        ? connectorVariables(namespace, descriptor.dwh.supported_transports)
-        : []),
+      ...connectorVariables(namespace, descriptor.dwh.supported_transports),
       ...evidenceVariables(namespace, descriptor),
     ],
   };

backend/src/workspaces/diagnostics.ts

@@ -12,7 +12,6 @@ import {
 import type { WorkspaceErrorCode } from "./types.js";
 import type { SemanticRuntimeConfig } from "./runtime-renderer.js";
 import type { AuthDiagnostics } from "../auth/diagnostics.js";
-import { workspaceVectorCollections } from "./vector-collections.js";
 
 export interface Diagnostic {
   level: "error" | "warning" | "info";
@@ -131,11 +130,6 @@ export interface DiagnosticAdapters {
   probeEmbedding(request: EmbeddingDiagnosticRequest): Promise<EmbeddingDiagnosticResult>;
 }
 
-export interface WorkspaceDiagnosticOptions {
-  writeProbe: boolean;
-  skipDwh?: boolean;
-}
-
 export const DEFAULT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS = 5_000;
 
 async function secretPresent(file: string): Promise<boolean> {
@@ -412,12 +406,12 @@ function numericBinding(binding: Record<string, string>, name: string): number |
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Workspace diagnoser supports only workspace schema version 4");
+    throw new Error("Workspace diagnoser supports only workspace schema version 3");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 4) {
-    throw new Error("Workspace diagnoser supports only workspace schema version 4");
+    || Reflect.get(metadata, "schema_version") !== 3) {
+    throw new Error("Workspace diagnoser supports only workspace schema version 3");
   }
 }
 
@@ -427,7 +421,6 @@ async function diagnoseValidatedWorkspace(
   adapters: DiagnosticAdapters,
   timeoutMs: number,
   semanticRuntime: SemanticRuntimeConfig,
-  skipDwh: boolean,
 ): Promise<ConnectorDiagnostics> {
   const evidenceField = descriptor.evidence?.source.type === "http"
     ? "evidence.source.authentication"
@@ -439,97 +432,88 @@ async function diagnoseValidatedWorkspace(
     variable,
   }));
   const diagnostics = [
-    ...(skipDwh
-      ? []
-      : [...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field))),
+    ...[...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field)),
     ...evidenceDiagnostics,
   ];
   if (diagnostics.length > 0) return { activatable: false, diagnostics };
 
+  const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs);
   let activatable = true;
-  if (!skipDwh) {
-    if (!descriptor.dwh) {
-      return { activatable: false, diagnostics: [diagnosticError("binding_missing", "dwh")] };
-    }
-    const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs);
-    const dwhValues = bindings.dwh.values;
-    const dwhField = (suffix: string) => bindingName(descriptor, suffix);
-    const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
-    let dwhRequest: ConnectorDiagnosticRequest | undefined;
-    if (bindings.dwh.transport === "rest_api") {
-      const diagnostic = descriptor.diagnostics?.dwh_rest;
-      const baseUrl = dwhValues[dwhField("BASE_URL")];
-      if (diagnostic && baseUrl) {
-        const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")];
-        if (diagnostic.auth === "none" || credentialFile !== undefined) {
-          dwhRequest = {
-            role: "dwh",
-            transport: "rest_api",
-            baseUrl,
-            credentialFile,
-            tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")],
-            resource: dwhResource,
-            timeoutMs: dwhTimeout,
-            signal: new AbortController().signal,
-            diagnostic,
-          };
-        }
-      }
-    } else if (bindings.dwh.transport === "postgres_direct") {
-      const host = dwhValues[dwhField("HOST")];
-      const port = numericBinding(dwhValues, dwhField("PORT"));
-      const user = dwhValues[dwhField("USER")];
-      const credentialFile = dwhValues[dwhField("PASSWORD_FILE")];
-      if (host && port && user && credentialFile) {
+  const dwhValues = bindings.dwh.values;
+  const dwhField = (suffix: string) => bindingName(descriptor, suffix);
+  const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
+  let dwhRequest: ConnectorDiagnosticRequest | undefined;
+  if (bindings.dwh.transport === "rest_api") {
+    const diagnostic = descriptor.diagnostics?.dwh_rest;
+    const baseUrl = dwhValues[dwhField("BASE_URL")];
+    if (diagnostic && baseUrl) {
+      const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")];
+      if (diagnostic.auth === "none" || credentialFile !== undefined) {
         dwhRequest = {
           role: "dwh",
-          transport: "postgres_direct",
-          host,
-          port,
-          user,
+          transport: "rest_api",
+          baseUrl,
           credentialFile,
           tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")],
           resource: dwhResource,
           timeoutMs: dwhTimeout,
           signal: new AbortController().signal,
+          diagnostic,
         };
       }
     }
-
-    if (!dwhRequest) {
-      diagnostics.push(diagnosticError("workspace_not_activatable"));
-      return { activatable: false, diagnostics };
-    }
-
-    try {
-      const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({
-        ...dwhRequest,
-        signal,
+  } else if (bindings.dwh.transport === "postgres_direct") {
+    const host = dwhValues[dwhField("HOST")];
+    const port = numericBinding(dwhValues, dwhField("PORT"));
+    const user = dwhValues[dwhField("USER")];
+    const credentialFile = dwhValues[dwhField("PASSWORD_FILE")];
+    if (host && port && user && credentialFile) {
+      dwhRequest = {
+        role: "dwh",
+        transport: "postgres_direct",
+        host,
+        port,
+        user,
+        credentialFile,
+        tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")],
+        resource: dwhResource,
         timeoutMs: dwhTimeout,
-      }));
-      if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) {
-        diagnostics.push(diagnosticError("connector_unavailable"));
-        activatable = false;
-      }
-    } catch {
-      diagnostics.push(diagnosticError("connector_unavailable"));
-      activatable = false;
+        signal: new AbortController().signal,
+      };
     }
   }
 
+  if (!dwhRequest) {
+    diagnostics.push(diagnosticError("workspace_not_activatable"));
+    return { activatable: false, diagnostics };
+  }
+
   try {
-    const expectedCollections = Object.values(workspaceVectorCollections(descriptor.workspace.id));
-    const vectors = await Promise.all(expectedCollections.map(async (collection) =>
-      await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
-        baseUrl: semanticRuntime.internalQdrantUrl,
-        collection,
-        timeoutMs,
-        signal,
-      }))));
-    if (vectors.some((vector, index) =>
-      vector.collection !== expectedCollections[index]
-      || vector.dimensions !== semanticRuntime.internalEmbeddingDimensions
-      || vector.distance !== "cosine")) {
+    const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({
+      ...dwhRequest,
+      signal,
+      timeoutMs: dwhTimeout,
+    }));
+    if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) {
+      diagnostics.push(diagnosticError("connector_unavailable"));
+      activatable = false;
+    }
+  } catch {
+    diagnostics.push(diagnosticError("connector_unavailable"));
+    activatable = false;
+  }
+
+  try {
+    const vector = await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
+      baseUrl: semanticRuntime.internalQdrantUrl,
+      collection: descriptor.semantic_index.vector_store.collection,
+      timeoutMs,
+      signal,
+    }));
+    const expected = descriptor.semantic_index.vector_store;
+    if (vector.collection !== expected.collection
+      || vector.dimensions !== expected.dimensions
+      || vector.distance !== expected.distance) {
       diagnostics.push(diagnosticError("semantic_index_incompatible"));
       activatable = false;
     }
@@ -545,8 +529,10 @@ async function diagnoseValidatedWorkspace(
       timeoutMs,
       signal,
     }));
-    if (!embedding.available
-      || embedding.dimensions !== semanticRuntime.internalEmbeddingDimensions) {
+    if (semanticRuntime.internalEmbeddingModel !== descriptor.semantic_index.embedding.model
+      || semanticRuntime.internalEmbeddingDimensions !== descriptor.semantic_index.embedding.dimensions
+      || !embedding.available
+      || embedding.dimensions !== descriptor.semantic_index.embedding.dimensions) {
       diagnostics.push(diagnosticError("semantic_index_incompatible"));
       activatable = false;
     }
@@ -583,18 +569,11 @@ export function createWorkspaceDiagnoser(
   return async function diagnose(
     workspace: WorkspaceDescriptor,
     bindings: RuntimeBindings,
-    diagnosticOptions: WorkspaceDiagnosticOptions,
+    _options: { writeProbe: boolean },
   ): Promise<ConnectorDiagnostics> {
     requireSupportedDescriptor(workspace);
     const descriptor = validateWorkspaceDescriptor(workspace);
-    return await diagnoseValidatedWorkspace(
-      descriptor,
-      bindings,
-      adapters,
-      timeoutMs,
-      semanticRuntime,
-      diagnosticOptions.skipDwh ?? false,
-    );
+    return await diagnoseValidatedWorkspace(descriptor, bindings, adapters, timeoutMs, semanticRuntime);
   };
 }
 

backend/src/workspaces/effective-config.ts

@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
 import { normalize } from "node:path";
 
 export interface CanonicalEffectiveConfig {
-  schemaVersion: 3;
+  schemaVersion: 1;
   dwh: CanonicalDwhConfig;
   vector: CanonicalVectorConfig;
   embedding: CanonicalEmbeddingConfig;
@@ -21,16 +21,12 @@ export interface CanonicalDwhConfig {
 }
 
 export interface CanonicalVectorConfig {
-  collections: {
-    reference: string;
-    memory: string;
-  };
+  collection: string;
   dimensions: number;
   distance: string;
 }
 
 export interface CanonicalEmbeddingConfig {
-  id: string;
   model: string;
   dimensions: number;
 }
@@ -123,10 +119,7 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
   if (!vector) {
     throw new TypeError("effective config is missing vector resources");
   }
-  const collections = asRecord(vector.collections);
-  if (!collections) {
-    throw new TypeError("effective config is missing vector collections");
-  }
+  const collection = requireString(vector, "collection");
   const semanticIndex = asRecord(rendered.semantic_index);
   const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined;
   const dimensions = vectorStore
@@ -135,14 +128,7 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
   const distance = vectorStore
     ? requireString(vectorStore, "distance")
     : (optionalString(vector, "distance") ?? "cosine");
-  return {
-    collections: {
-      reference: requireString(collections, "reference"),
-      memory: requireString(collections, "memory"),
-    },
-    dimensions,
-    distance,
-  };
+  return { collection, dimensions, distance };
 }
 
 function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbeddingConfig {
@@ -151,10 +137,8 @@ function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbed
   if (!embeddings) {
     throw new TypeError("effective config is missing embedding resources");
   }
-  const model = requireString(embeddings, "model");
   return {
-    id: optionalString(embeddings, "id") ?? `ollama/${model}`,
-    model,
+    model: requireString(embeddings, "model"),
     dimensions: requireNumber(embeddings, "dimensions"),
   };
 }
@@ -182,7 +166,7 @@ export function buildCanonicalEffectiveConfig(renderedConfig: unknown): Canonica
     throw new TypeError("effective config requires a rendered configuration object");
   }
   return {
-    schemaVersion: 3,
+    schemaVersion: 1,
     dwh: buildDwhConfig(rendered),
     vector: buildVectorConfig(rendered),
     embedding: buildEmbeddingConfig(rendered),
@@ -233,20 +217,3 @@ export function configFingerprint(renderedConfig: unknown): string {
 export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string {
   return sha256(effectiveConfigIdentity(workspaceId, renderedConfig));
 }
-
-/**
- * Fingerprint every non-Catalog input consumed by complete preprocessing. The immutable Git
- * revision covers the workspace descriptor and its revision-pinned Evidence tree; the effective
- * configuration identity covers the Catalog-derived DWH binding and semantic runtime contract.
- */
-export function preprocessingInputFingerprint(
-  workspaceId: string,
-  workspaceRevision: string,
-  renderedConfig: unknown,
-): string {
-  return sha256(JSON.stringify({
-    workspaceId,
-    workspaceRevision,
-    effectiveConfigIdentity: effectiveConfigIdentity(workspaceId, renderedConfig),
-  }));
-}

backend/src/workspaces/evidence/preprocessing.ts

@@ -171,14 +171,6 @@ export async function continueEvidencePreprocessing(
   const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
   if (policy) return { ...policy, ...jobResult(request.job) };
   if (!request.job.completedStages.includes("evidence")) {
-    const preflight = await deps.evidencePreflight();
-    if (!preflight.ok) {
-      return {
-        status: "failed",
-        code: preflight.code,
-        ...jobResult(request.job),
-      };
-    }
     return await runEvidenceStage(request, deps);
   }
   return { status: "unchanged", code: "ok", ...jobResult(request.job) };

backend/src/workspaces/preprocessing-service.ts

@@ -1,19 +1,22 @@
-import { randomBytes } from "node:crypto";
-import { renameSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
+import { createHash, randomBytes } from "node:crypto";
+import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import {
   continueEvidencePreprocessing,
+  preprocessEvidence as runEvidencePreprocessing,
   type EvidencePreprocessingDependencies,
   type EvidencePreprocessingOutcome,
 } from "./evidence/preprocessing.js";
 import type { WorkspaceDescriptor } from "./schema.js";
 import {
   PreprocessingStateStore,
+  type FkReviewRecord,
   type PreprocessingJobState,
+  type SessionInventoryRow,
 } from "./preprocessing-state.js";
 import type { DeterministicRuntimeConfigLease } from "./runtime-config-lease.js";
-import { buildCatalogMetadataSnapshot } from "../catalog/metadata-snapshot.js";
-import type { CatalogRepository } from "../catalog/types.js";
+import { readAnnotationsSync } from "./annotations-sync.js";
+import { reconcileCollection } from "./qdrant-collection.js";
 
 export interface WorkspaceOperationResult {
   schemaVersion: 1;
@@ -24,7 +27,7 @@ export interface WorkspaceOperationResult {
     | "preprocessing_resume_mismatch" | "manual_review_required"
     | "evidence_materialization_required" | "effective_config_mismatch"
     | "semantic_index_incompatible" | "annotation_invalid"
-    | "egress_policy_refused" | "catalog_not_ready" | "preprocessing_clear_failed";
+    | "egress_policy_refused";
   workspaceId: string;
   workspaceRevision: string;
   descriptorBlob: string;
@@ -66,6 +69,7 @@ export interface WorkspacePreprocessingServiceDeps {
   dataRoot: string;
   acquireActiveRuntime(workspaceId: string): Promise<ActiveRuntime>;
   runChild(request: ChildProcessRequest): Promise<ChildProcessResult>;
+  listSessions(workspaceId: string): Promise<readonly SessionInventoryRow[]>;
   semanticPreflight(workspace: WorkspaceDescriptor): Promise<
     { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
   >;
@@ -73,7 +77,6 @@ export interface WorkspacePreprocessingServiceDeps {
     { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
   >;
   httpPrivateHostAllowlist?: readonly string[];
-  catalogRepository?: CatalogRepository;
 }
 
 interface RunScope {
@@ -82,6 +85,10 @@ interface RunScope {
   job: PreprocessingJobState;
 }
 
+function digest(value: string | Buffer): string {
+  return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}
+
 function baseResult(
   runtime: ActiveRuntime,
   operation: string,
@@ -108,6 +115,41 @@ function baseResult(
 export class WorkspacePreprocessingService {
   constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
 
+
+  async vectorInspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
+    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
+    const collection = runtime.workspace.semantic_index.vector_store.collection;
+    const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" });
+    if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] });
+    const body = await res.json() as any;
+    const info = body?.result;
+    const vectors = info?.config?.params?.vectors;
+    return baseResult(runtime, "vector inspect", "succeeded", "ok", {
+      counts: { dimensions: vectors?.size ?? 0 },
+      warnings: [`collection=${collection} distance=${vectors?.distance ?? "unknown"}`],
+    });
+  }
+
+  async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise<WorkspaceOperationResult> {
+    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
+    const collection = runtime.workspace.semantic_index.vector_store.collection;
+    if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) {
+      return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] });
+    }
+    const q = `${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`;
+    const del = await fetch(q, { method: "DELETE" });
+    if (!del.ok && del.status !== 404) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection delete failed"] });
+    // Recreate the complete contract (dimensions + distance + the 8 required keyword indexes).
+    const recreated = await reconcileCollection({
+      baseUrl: runtime.configLease.semanticQdrantUrl,
+      collection,
+      dimensions: runtime.workspace.semantic_index.vector_store.dimensions,
+      distance: runtime.workspace.semantic_index.vector_store.distance,
+      mode: "self_heal",
+    });
+    if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] });
+    return baseResult(runtime, "vector rebuild", "succeeded", "ok", { warnings: [`recreated collection=${collection}`] });
+  }
   async inspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
     try {
       const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
@@ -132,158 +174,232 @@ export class WorkspacePreprocessingService {
     }
   }
 
-  async run(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
-    if (!this.deps.catalogRepository) {
-      const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-      return baseResult(runtime, "preprocess run", "failed", "catalog_not_ready");
+  async preprocessDwh(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
+    const scope = await this.startRun(options.workspaceId, "preprocess dwh", options.resumeRunId);
+    if (scope.job.completedStages.includes("dwh")) {
+      return baseResult(scope.runtime, "preprocess dwh", "unchanged", "ok", {
+        runId: scope.job.runId,
+        childRuns: scope.job.childRuns,
+        completedStages: [...scope.job.completedStages],
+      });
     }
-    return await this.runFromCatalog(options);
+    const payload = await this.runJsonStage(scope.runtime, [
+      "preprocess", "dwh", "--steps", "introspect,lsh",
+      ...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
+      "--json", "-c", "/dev/fd/3",
+    ]);
+    const childRun = this.requireRunId(payload.run_id);
+    scope.job.childRuns.dwh = childRun;
+    if (!scope.job.completedStages.includes("dwh")) scope.job.completedStages.push("dwh");
+    this.state(scope.runtime.workspaceId).writeJob(scope.job);
+    return baseResult(scope.runtime, "preprocess dwh", "succeeded", "ok", {
+      runId: scope.job.runId,
+      childRuns: { ...scope.job.childRuns },
+      completedStages: [...scope.job.completedStages],
+    });
   }
 
-  async clear(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
+  async suggestFks(options: {
+    workspaceId: string;
+    fromSql?: ReadonlyArray<{ name: string; sql: string }>;
+    assume?: readonly string[];
+    resumeRunId?: string;
+  }): Promise<WorkspaceOperationResult> {
+    const scope = await this.startRun(options.workspaceId, "schema suggest-fks", options.resumeRunId);
+    return await this.runSuggestStage(scope, options.fromSql ?? [], options.assume ?? []);
+  }
+
+  async checkSchema(options: {
+    workspaceId: string;
+    annotationsYaml?: string;
+    reviewedCandidatesDigest?: string;
+  }): Promise<WorkspaceOperationResult> {
     const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-    const repository = this.deps.catalogRepository;
-    if (!repository) return baseResult(runtime, "preprocess clear", "failed", "catalog_not_ready");
-    const cleared = await repository.clearPreprocessing(runtime.workspaceId);
-    if (cleared.kind !== "cleared") {
-      return baseResult(
-        runtime,
-        "preprocess clear",
-        "failed",
-        cleared.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
-      );
+    const state = this.state(runtime.workspaceId);
+    if ((options.annotationsYaml === undefined) !== (options.reviewedCandidatesDigest === undefined)) {
+      return baseResult(runtime, "schema check", "failed", "annotation_invalid");
     }
-    const result = await this.deps.runChild({
-      argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
-      configPath: runtime.configLease.path,
+    if (options.reviewedCandidatesDigest === undefined) {
+      const payload = await this.runJsonStage(runtime, ["schema", "check", "--json", "-c", "/dev/fd/3"]);
+      return baseResult(runtime, "schema check", Number(payload.orphan_count ?? 0) === 0 ? "succeeded" : "failed", Number(payload.orphan_count ?? 0) === 0 ? "ok" : "annotation_invalid");
+    }
+    const reviewedCandidatesDigest = options.reviewedCandidatesDigest;
+    const runId = this.findRunIdByCandidateDigest(state, reviewedCandidatesDigest);
+    if (!runId) return baseResult(runtime, "schema check", "failed", "annotation_invalid");
+    const request = await this.withStagedInputs(runtime.workspaceId, [
+      { flag: "--annotations", name: "annotations.yaml", contents: options.annotationsYaml! },
+    ], async (argv) => await this.runJsonStage(runtime, [
+      "schema", "check", ...argv,
+      "--reviewed-candidates", reviewedCandidatesDigest,
+      "--json", "-c", "/dev/fd/3",
+    ]));
+    if (request.reviewed_candidates_digest !== reviewedCandidatesDigest || typeof request.annotations_digest !== "string") {
+      return baseResult(runtime, "schema check", "failed", "annotation_invalid", { runId });
+    }
+    // P5 supersedes the host-file FK review: schema check is read-only validation and never
+    // records a review. Only `schema accept` records a human review for the curated Git blob.
+    return baseResult(runtime, "schema check", "succeeded", "ok", { runId });
+  }
+
+  async acceptSchema(options: { workspaceId: string; runId: string; yes?: boolean }): Promise<WorkspaceOperationResult> {
+    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
+    const state = this.state(runtime.workspaceId);
+    if (options.yes !== true) {
+      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
+        runId: options.runId,
+        warnings: ["accept requires --yes"],
+      });
+    }
+    if (!/^[0-9a-f]{32}$/.test(options.runId)) {
+      return baseResult(runtime, "schema accept", "failed", "annotation_invalid");
+    }
+    const candidate = state.readFkCandidates(options.runId);
+    if (candidate === undefined) {
+      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
+        runId: options.runId,
+        warnings: ["candidate run is unavailable"],
+      });
+    }
+    const synced = readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision);
+    if (synced === undefined || synced.contents.toString("utf8").trim() === "") {
+      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
+        runId: options.runId,
+        warnings: ["curated annotations are not synchronized"],
+      });
+    }
+    // The harness parser validates the curated blob against the physical schema; the recorded
+    // candidate digest must round-trip and the blob digest must match the synced destination.
+    const payload = await this.runJsonStage(runtime, [
+      "schema", "check", "--reviewed-candidates", candidate.digest, "--json", "-c", "/dev/fd/3",
+    ]);
+    if (payload.annotations_digest !== synced.contentDigest
+      || payload.reviewed_candidates_digest !== candidate.digest
+      || Number(payload.orphan_count ?? 0) !== 0) {
+      return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { runId: options.runId });
+    }
+    const review = state.writeFkReview(options.runId, {
+      reviewedCandidatesDigest: candidate.digest,
+      annotationsDigest: synced.contentDigest,
+      workspaceRevision: runtime.workspaceRevision,
+      blobId: synced.blobId,
     });
-    if (result.exitCode !== 0) {
-      return baseResult(runtime, "preprocess clear", "failed", "preprocessing_clear_failed");
-    }
-    const payload = JSON.parse(result.stdout) as Record<string, unknown>;
-    return baseResult(runtime, "preprocess clear", "succeeded", "ok", {
-      counts: this.numberRecord(payload.counts),
+    const job = state.readJob(options.runId);
+    job.reviewDigest = review.digest;
+    if (!job.completedStages.includes("fk_review")) job.completedStages.push("fk_review");
+    state.writeJob(job);
+    return baseResult(runtime, "schema accept", "succeeded", "ok", {
+      runId: options.runId,
+      completedStages: [...job.completedStages],
+      artifactIdentities: [
+        { kind: "fk_review", digest: review.digest },
+        { kind: "annotations", digest: synced.contentDigest },
+      ],
     });
   }
 
-  private async runFromCatalog(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
-    const scope = await this.startRun(options.workspaceId);
-    const repository = this.deps.catalogRepository!;
-    const fingerprint = scope.runtime.configLease.inputFingerprint;
-    const started = await repository.beginPreprocessing(scope.runtime.workspaceId, fingerprint);
-    if (started.kind !== "started") {
-      scope.job.status = "failed";
-      scope.state.writeJob(scope.job);
-      return baseResult(
-        scope.runtime,
-        "preprocess run",
-        "failed",
-        started.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
-        { warnings: [`catalog=${started.kind}`] },
-      );
+  async indexSchema(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
+    const scope = await this.startRun(options.workspaceId, "index-schema", options.resumeRunId);
+    const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
+    if (!semantic.ok) return baseResult(scope.runtime, "index-schema", "failed", semantic.code, { runId: scope.job.runId });
+    if (scope.job.completedStages.includes("schema_index")) {
+      return baseResult(scope.runtime, "index-schema", "unchanged", "ok", {
+        runId: scope.job.runId,
+        completedStages: [...scope.job.completedStages],
+      });
     }
+    const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
+    const counts = this.numberRecord(payload.counts);
+    scope.job.completedStages.push("schema_index");
+    this.state(scope.runtime.workspaceId).writeJob(scope.job);
+    return baseResult(scope.runtime, "index-schema", "succeeded", "ok", {
+      runId: scope.job.runId,
+      completedStages: [...scope.job.completedStages],
+      counts,
+    });
+  }
 
-    const revision = started.database.metadataContentRevision;
-    let finished = false;
-    let failureCode = "catalog_snapshot_failed";
-    try {
-      const snapshot = await buildCatalogMetadataSnapshot(
-        repository,
-        scope.runtime.workspaceId,
-        revision,
-      );
-      const snapshotPath = this.publishCatalogSnapshot(
-        scope.runtime.workspaceId,
-        JSON.stringify(snapshot),
-      );
-      this.completeStages(scope, "catalog_snapshot");
-      const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
-      if (!semantic.ok) {
-        await repository.finishPreprocessing(
-          scope.runtime.workspaceId,
-          revision,
-          fingerprint,
-          { status: "failed", errorCode: semantic.code },
-        );
-        scope.job.status = "failed";
-        scope.state.writeJob(scope.job);
-        finished = true;
-        return baseResult(scope.runtime, "preprocess run", "failed", semantic.code);
-      }
+  async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
+    const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId);
+    const outcome = await runEvidencePreprocessing(
+      {
+        evidence: scope.runtime.workspace.evidence,
+        job: scope.job,
+        dryRun: options.dryRun,
+        httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
+      },
+      this.evidenceDependencies(scope),
+    );
+    return this.evidenceResult(scope, "preprocess evidence", outcome);
+  }
 
-      failureCode = "schema_index_failed";
+  async run(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
+    const scope = await this.startRun(options.workspaceId, "preprocess run", options.resumeRunId);
+    if (!scope.job.completedStages.includes("dwh")) {
       const payload = await this.runJsonStage(scope.runtime, [
-        "preprocess",
-        "catalog",
-        "--catalog-metadata",
-        snapshotPath,
-        "--json",
-        "-c",
-        "/dev/fd/3",
+        "preprocess", "dwh", "--steps", "introspect,lsh",
+        ...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
+        "--json", "-c", "/dev/fd/3",
       ]);
-      this.completeStages(scope, "catalog_metadata", "lsh", "schema_index");
-      failureCode = "evidence_preprocessing_failed";
-      const outcome = await continueEvidencePreprocessing(
-        {
-          evidence: scope.runtime.workspace.evidence,
-          job: scope.job,
-          httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
-          priorCounts: this.numberRecord(payload.counts),
-        },
-        this.evidenceDependencies(scope),
-      );
-      if (!["succeeded", "unchanged"].includes(outcome.status)) {
-        await repository.finishPreprocessing(
-          scope.runtime.workspaceId,
-          revision,
-          fingerprint,
-          { status: "failed", errorCode: outcome.code },
-        );
-        scope.job.status = "failed";
-        scope.state.writeJob(scope.job);
-        finished = true;
-        return this.evidenceResult(scope, outcome);
-      }
-      // Evidence has been published. The only remaining operation is the atomic Catalog commit.
-      this.completeStages(scope, "evidence");
-      const completed = await repository.finishPreprocessing(
-        scope.runtime.workspaceId,
-        revision,
-        fingerprint,
-        { status: "succeeded" },
-      );
-      if (!completed) throw new Error("catalog preprocessing completion lost its lease");
-      scope.job.status = "succeeded";
-      scope.state.writeJob(scope.job);
-      finished = true;
-      return this.evidenceResult(scope, outcome);
-    } catch (error) {
-      if (!finished) {
-        await repository.finishPreprocessing(
-          scope.runtime.workspaceId,
-          revision,
-          fingerprint,
-          { status: "failed", errorCode: failureCode },
-        );
-      }
-      scope.job.status = "failed";
-      scope.state.writeJob(scope.job);
-      throw error;
+      scope.job.childRuns.dwh = this.requireRunId(payload.run_id);
+      scope.job.completedStages.push("dwh");
+      this.state(scope.runtime.workspaceId).writeJob(scope.job);
     }
+    if (!scope.job.completedStages.includes("fk_suggest")) {
+      const suggest = await this.runSuggestStage(scope, [], []);
+      if (suggest.code === "manual_review_required") return suggest;
+    }
+    const candidate = this.state(scope.runtime.workspaceId).readFkCandidates(scope.job.runId);
+    if (candidate && !scope.job.completedStages.includes("fk_review")) {
+      // P5: continuation requires a review accepted for this candidate whose accepted blob digest
+      // equals the current revision's synced annotations. A revision change (or a missing curated
+      // blob) therefore records a new review checkpoint instead of silently reusing the old one.
+      const accepted = this.findAcceptedReviewForDigest(scope.runtime.workspaceId, candidate.digest);
+      const currentDigest = this.currentAnnotationsDigest(scope.runtime);
+      if (accepted === undefined || currentDigest === undefined || accepted.annotationsDigest !== currentDigest) {
+        return baseResult(scope.runtime, "preprocess run", "blocked", "manual_review_required", {
+          runId: scope.job.runId,
+          childRuns: { ...scope.job.childRuns },
+          completedStages: [...scope.job.completedStages],
+          artifactIdentities: [{ kind: "fk_candidates", digest: candidate.digest }],
+        });
+      }
+      scope.job.reviewDigest = accepted.reviewedCandidatesDigest;
+      scope.job.completedStages.push("fk_review");
+      this.state(scope.runtime.workspaceId).writeJob(scope.job);
+    }
+    const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
+    if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId });
+    let schemaCounts: Record<string, number> | undefined;
+    if (!scope.job.completedStages.includes("schema_index")) {
+      const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
+      scope.job.completedStages.push("schema_index");
+      this.state(scope.runtime.workspaceId).writeJob(scope.job);
+      schemaCounts = this.numberRecord(payload.counts);
+    }
+    const outcome = await continueEvidencePreprocessing(
+      {
+        evidence: scope.runtime.workspace.evidence,
+        job: scope.job,
+        httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
+        priorCounts: schemaCounts,
+      },
+      this.evidenceDependencies(scope),
+    );
+    return this.evidenceResult(scope, "preprocess run", outcome);
   }
 
-  private async startRun(workspaceId: string): Promise<RunScope> {
+  private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise<RunScope> {
     const runtime = await this.deps.acquireActiveRuntime(workspaceId);
     const state = this.state(runtime.workspaceId);
+    await state.assertSessionInventoryCompatible(runtime.workspaceRevision, await this.deps.listSessions(runtime.workspaceId));
     const job = await state.beginJob({
-      operation: "preprocess run",
+      operation,
+      runId: resumeRunId,
       workspaceRevision: runtime.workspaceRevision,
       descriptorBlob: runtime.descriptorBlob,
       catalogBlob: runtime.catalogBlob,
       configDigest: runtime.configLease.configDigest,
       bindingDigest: runtime.configLease.bindingDigest,
-      embeddingId: runtime.configLease.effectiveConfig.embedding.id,
-      embeddingDimensions: runtime.configLease.effectiveConfig.embedding.dimensions,
     });
     return { runtime, state, job };
   }
@@ -292,28 +408,6 @@ export class WorkspacePreprocessingService {
     return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId });
   }
 
-  private completeStages(scope: RunScope, ...stages: string[]): void {
-    for (const stage of stages) {
-      if (!scope.job.completedStages.includes(stage)) scope.job.completedStages.push(stage);
-    }
-    scope.state.writeJob(scope.job);
-  }
-
-  private publishCatalogSnapshot(workspaceId: string, contents: string): string {
-    const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing");
-    mkdirSync(root, { recursive: true, mode: 0o700 });
-    const target = join(root, "catalog-metadata.json");
-    const staging = join(root, `.catalog-metadata-${randomBytes(6).toString("hex")}.json`);
-    try {
-      writeFileSync(staging, contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
-      renameSync(staging, target);
-      return target;
-    } catch (error) {
-      rmSync(staging, { force: true });
-      throw error;
-    }
-  }
-
   private evidenceDependencies(scope: RunScope): EvidencePreprocessingDependencies {
     return {
       runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
@@ -326,10 +420,50 @@ export class WorkspacePreprocessingService {
 
   private evidenceResult(
     scope: RunScope,
+    operation: "preprocess evidence" | "preprocess run",
     outcome: EvidencePreprocessingOutcome,
   ): WorkspaceOperationResult {
     const { status, code, ...extra } = outcome;
-    return baseResult(scope.runtime, "preprocess run", status, code, extra);
+    return baseResult(scope.runtime, operation, status, code, extra);
+  }
+
+  private async runSuggestStage(
+    scope: RunScope,
+    fromSql: ReadonlyArray<{ name: string; sql: string }>,
+    assume: readonly string[],
+  ): Promise<WorkspaceOperationResult> {
+    const payload = await this.withStagedInputs(scope.runtime.workspaceId, fromSql.map((entry) => ({
+      flag: "--from-sql",
+      name: entry.name,
+      contents: entry.sql,
+    })), async (stagedArgv) => await this.runJsonStage(scope.runtime, [
+      "schema", "suggest-fks", ...stagedArgv,
+      ...assume.flatMap((value) => ["--assume", value]),
+      "--json", "-c", "/dev/fd/3",
+    ]));
+    const candidateCount = Number(payload.candidate_count ?? 0);
+    const candidateYaml = typeof payload.candidate_yaml === "string" ? payload.candidate_yaml : "";
+    let artifactIdentities: Array<{ kind: string; digest: string }> | undefined;
+    if (candidateCount > 0) {
+      const persisted = this.state(scope.runtime.workspaceId).writeFkCandidates(scope.job.runId, candidateYaml);
+      scope.job.candidateDigest = persisted.digest;
+      artifactIdentities = [{ kind: "fk_candidates", digest: persisted.digest }];
+    }
+    if (!scope.job.completedStages.includes("fk_suggest")) scope.job.completedStages.push("fk_suggest");
+    this.state(scope.runtime.workspaceId).writeJob(scope.job);
+    const resultExtra = {
+      runId: scope.job.runId,
+      completedStages: [...scope.job.completedStages],
+      ...(artifactIdentities ? { artifactIdentities } : {}),
+      ...(candidateYaml.length > 0 ? { suggestedFksYaml: candidateYaml } : {}),
+    };
+    if (candidateCount > 0) {
+      return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "blocked", "manual_review_required", {
+        ...resultExtra,
+        childRuns: { ...scope.job.childRuns },
+      });
+    }
+    return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "succeeded", "ok", resultExtra);
   }
 
   private async runJsonStage(runtime: ActiveRuntime, argv: string[]): Promise<Record<string, unknown>> {
@@ -348,5 +482,54 @@ export class WorkspacePreprocessingService {
     return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([key, nested]) => [key, Number(nested)]));
   }
 
+  private async withStagedInputs<T>(
+    workspaceId: string,
+    inputs: ReadonlyArray<{ flag: string; name: string; contents: string }>,
+    fn: (argv: string[]) => Promise<T>,
+  ): Promise<T> {
+    if (inputs.length === 0) return await fn([]);
+    const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing", `.stage-${randomBytes(6).toString("hex")}`);
+    mkdirSync(root, { recursive: true, mode: 0o700 });
+    const argv: string[] = [];
+    const paths: string[] = [];
+    try {
+      for (const input of inputs) {
+        const path = join(root, input.name);
+        writeFileSync(path, input.contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
+        paths.push(path);
+        argv.push(input.flag, path);
+      }
+      return await fn(argv);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  }
+
+  private currentAnnotationsDigest(runtime: ActiveRuntime): string | undefined {
+    return readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision)?.contentDigest;
+  }
+
+  private findAcceptedReviewForDigest(
+    workspaceId: string,
+    digestValue: string,
+  ): FkReviewRecord | undefined {
+    const state = this.state(workspaceId);
+    for (const entry of readdirSync(state.fkReviewsDirectory(), { withFileTypes: true })) {
+      if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.json$/.test(entry.name)) continue;
+      const runId = entry.name.slice(0, -".json".length);
+      const review = state.readFkReview(runId);
+      if (review && review.reviewedCandidatesDigest === digestValue) return review;
+    }
+    return undefined;
+  }
+
+  private findRunIdByCandidateDigest(state: PreprocessingStateStore, digestValue: string): string | undefined {
+    for (const entry of readdirSync(state.fkCandidatesDirectory(), { withFileTypes: true })) {
+      if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.yaml$/.test(entry.name)) continue;
+      const runId = entry.name.slice(0, -".yaml".length);
+      if (state.readFkCandidates(runId)?.digest === digestValue) return runId;
+    }
+    return undefined;
+  }
 
 }

backend/src/workspaces/preprocessing-state.ts

@@ -44,13 +44,11 @@ export interface BeginPreprocessingJobOptions {
   catalogBlob: string;
   configDigest: string;
   bindingDigest: string;
-  embeddingId: string;
-  embeddingDimensions: number;
   runId?: string;
 }
 
 export interface PreprocessingJobState {
-  schemaVersion: 2;
+  schemaVersion: 1;
   runId: string;
   operation: string;
   workspaceId: string;
@@ -59,8 +57,6 @@ export interface PreprocessingJobState {
   catalogBlob: string;
   configDigest: string;
   bindingDigest: string;
-  embeddingId: string;
-  embeddingDimensions: number;
   completedStages: string[];
   childRuns: Record<string, string>;
   status: "active" | "succeeded" | "blocked" | "failed";
@@ -150,7 +146,7 @@ function decodeJob(value: unknown): PreprocessingJobState {
   }
   const record = value as Record<string, unknown>;
   if (
-    record.schemaVersion !== 2
+    record.schemaVersion !== 1
     || typeof record.runId !== "string"
     || typeof record.operation !== "string"
     || typeof record.workspaceId !== "string"
@@ -159,8 +155,6 @@ function decodeJob(value: unknown): PreprocessingJobState {
     || typeof record.catalogBlob !== "string"
     || typeof record.configDigest !== "string"
     || typeof record.bindingDigest !== "string"
-    || typeof record.embeddingId !== "string"
-    || typeof record.embeddingDimensions !== "number"
     || !Array.isArray(record.completedStages)
     || typeof record.childRuns !== "object" || record.childRuns === null || Array.isArray(record.childRuns)
     || !["active", "succeeded", "blocked", "failed"].includes(String(record.status))
@@ -198,7 +192,6 @@ export class PreprocessingStateStore {
   runtimeConfigDirectory(): string { return join(this.root, "runtime-config"); }
   runtimeConfigManifestDirectory(): string { return join(this.root, "runtime-config-manifests"); }
   jobsDirectory(): string { return join(this.root, "jobs"); }
-  latestJobPath(): string { return join(this.root, "latest-job.json"); }
   fkCandidatesDirectory(): string { return join(this.root, "fk-candidates"); }
   fkReviewsDirectory(): string { return join(this.root, "fk-reviews"); }
   jobPath(runId: string): string { return join(this.jobsDirectory(), `${validateRunId(runId)}.json`); }
@@ -282,15 +275,13 @@ export class PreprocessingStateStore {
         || existing.catalogBlob !== options.catalogBlob
         || existing.configDigest !== options.configDigest
         || existing.bindingDigest !== options.bindingDigest
-        || existing.embeddingId !== options.embeddingId
-        || existing.embeddingDimensions !== options.embeddingDimensions
       ) {
         throw new PreprocessingStateError(
           "preprocessing_resume_mismatch",
           "Workspace preprocessing resume no longer matches the pinned revision",
         );
       }
-      return this.writeJob(existing);
+      return existing;
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
         if (error instanceof PreprocessingStateError) throw error;
@@ -304,7 +295,7 @@ export class PreprocessingStateStore {
       }
     }
     const job: PreprocessingJobState = {
-      schemaVersion: 2,
+      schemaVersion: 1,
       runId,
       operation: options.operation,
       workspaceId: this.options.workspaceId,
@@ -313,36 +304,23 @@ export class PreprocessingStateStore {
       catalogBlob: options.catalogBlob,
       configDigest: options.configDigest,
       bindingDigest: options.bindingDigest,
-      embeddingId: options.embeddingId,
-      embeddingDimensions: options.embeddingDimensions,
       completedStages: [],
       childRuns: {},
       status: "active",
     };
-    return this.writeJob(job);
+    writeAtomicFile(path, `${JSON.stringify(job)}
+`, 0o600);
+    return job;
   }
 
   readJob(runId: string): PreprocessingJobState {
     return decodeJob(JSON.parse(readTrustedFile(this.jobPath(runId))));
   }
 
-  readLatestJob(): PreprocessingJobState | undefined {
-    try {
-      return decodeJob(JSON.parse(readTrustedFile(this.latestJobPath())));
-    } catch (error) {
-      if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
-      throw error;
-    }
-  }
-
   writeJob(job: PreprocessingJobState): PreprocessingJobState {
     this.ensureLayout();
-    const contents = `${JSON.stringify(job)}
-`;
-    writeAtomicFile(this.jobPath(job.runId), contents, 0o600);
-    // This fixed pointer is the sole status surface for operators. Per-run files remain an
-    // internal resume mechanism and are never exposed as history.
-    writeAtomicFile(this.latestJobPath(), contents, 0o600);
+    writeAtomicFile(this.jobPath(job.runId), `${JSON.stringify(job)}
+`, 0o600);
     return job;
   }
 

backend/src/workspaces/registry.ts

@@ -560,7 +560,7 @@ export class WorkspaceRegistry {
             });
           }
         }
-        const collection = workspace.workspace.id;
+        const collection = workspace.semantic_index.vector_store.collection;
         const owner = collectionOwners.get(collection);
         if (owner !== undefined) {
           throw new Error(`duplicate qdrant collection ownership: ${collection} (${owner}, ${id})`);

backend/src/workspaces/runtime-config-lease.ts

@@ -23,7 +23,7 @@ import {
   canonicalEffectiveConfigJson,
   configFingerprint,
   effectiveConfigIdentity,
-  preprocessingInputFingerprint,
+  inputFingerprint,
   type CanonicalEffectiveConfig,
 } from "./effective-config.js";
 import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
@@ -41,8 +41,6 @@ import {
 } from "./runtime-renderer.js";
 import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js";
 import type { WorkspaceRegistryConfig } from "./types.js";
-import { resolveCatalogRuntimeBinding } from "../catalog/runtime-binding.js";
-import type { WorkspaceDatabase } from "../catalog/types.js";
 
 export interface RuntimeConfigLease {
   path: string;
@@ -248,6 +246,8 @@ function readSnapshotWorkspace(snapshotPath: string): {
 function runtimePaths(
   dataRoot: string,
   workspaceId: string,
+  workspaceRevision?: string,
+  effectiveRelationshipsPath?: string,
 ): RuntimePaths {
   if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
   const root = join(dataRoot, "sessions", workspaceId);
@@ -256,7 +256,12 @@ function runtimePaths(
     artifacts: join(root, "artifacts"),
     indexes: join(root, "indexes"),
     memory: join(root, "memory"),
-    catalog_metadata_snapshot: join(root, "preprocessing", "catalog-metadata.json"),
+    ...(workspaceRevision === undefined
+      ? {}
+      : { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }),
+    ...(effectiveRelationshipsPath === undefined
+      ? {}
+      : { effective_relationships: effectiveRelationshipsPath }),
   };
 }
 
@@ -304,32 +309,19 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
   dataRoot: string;
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
+  effectiveRelationshipsPath?: string;
   workspaceSecretStore?: WorkspaceSecretStore;
-  catalogDatabase?: WorkspaceDatabase;
 }): RenderedWorkspaceRuntime {
-  if (options.catalogDatabase && !options.workspaceSecretStore) {
-    throw new Error("Catalog runtime binding requires the workspace secret store");
-  }
-  const catalogLease = options.catalogDatabase === undefined
-    ? undefined
-    : resolveCatalogRuntimeBinding({
-      workspace: options.workspace,
-      database: options.catalogDatabase,
-      environment: process.env,
-      secretRoots: options.secretRoots,
-      secretStore: options.workspaceSecretStore!,
-    });
-  const runtimeWorkspace = catalogLease?.workspace ?? options.workspace;
-  const secretLease = catalogLease !== undefined || options.workspaceSecretStore === undefined
+  const secretLease = options.workspaceSecretStore === undefined
     ? undefined
     : resolveRuntimeBindingsWithWorkspaceSecrets(
-      runtimeWorkspace,
+      options.workspace,
       process.env,
       options.secretRoots,
       options.workspaceSecretStore,
     );
-  const bindings = catalogLease?.bindings ?? secretLease?.bindings
-    ?? resolveRuntimeBindings(runtimeWorkspace, process.env, options.secretRoots);
+  const bindings = secretLease?.bindings
+    ?? resolveRuntimeBindings(options.workspace, process.env, options.secretRoots);
   const overlay = installationOverlay(options.harnessDir, options.configPath);
   const context: RuntimeRenderContext = {
     workspaceId: options.workspaceId,
@@ -342,26 +334,32 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
       workspaceId: options.workspaceId,
       workspaceRevision: options.workspaceRevision,
       revisionContentRoot: options.revisionContentRoot,
-      runtimePaths: runtimePaths(options.dataRoot, options.workspaceId),
+      runtimePaths: runtimePaths(
+        options.dataRoot,
+        options.workspaceId,
+        options.workspaceRevision,
+        options.effectiveRelationshipsPath,
+      ),
       installationOverlay: overlay,
       bindings,
       bindingDigest: stableBindingDigest(bindings),
       semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
-      releaseSecrets: () => {
-        catalogLease?.release();
-        secretLease?.release();
-      },
+      releaseSecrets: () => secretLease?.release(),
       renderedConfig: renderRuntimeConfig(
-        runtimeWorkspace,
+        options.workspace,
         bindings,
-        runtimePaths(options.dataRoot, options.workspaceId),
+        runtimePaths(
+          options.dataRoot,
+          options.workspaceId,
+          options.workspaceRevision,
+          options.effectiveRelationshipsPath,
+        ),
         context,
         overlay,
         options.semanticRuntime,
       ),
     };
   } catch (error) {
-    catalogLease?.release();
     secretLease?.release();
     throw error;
   }
@@ -374,8 +372,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
   dataRoot: string;
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
+  effectiveRelationshipsPath?: string;
   workspaceSecretStore?: WorkspaceSecretStore;
-  catalogDatabase?: WorkspaceDatabase;
 }): RenderedWorkspaceRuntime {
   const snapshot = readSnapshotWorkspace(options.snapshotPath);
   return renderWorkspaceRuntimeFromWorkspace({
@@ -388,8 +386,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
     dataRoot: options.dataRoot,
     secretRoots: options.secretRoots,
     semanticRuntime: options.semanticRuntime,
+    effectiveRelationshipsPath: options.effectiveRelationshipsPath,
     workspaceSecretStore: options.workspaceSecretStore,
-    catalogDatabase: options.catalogDatabase,
   });
 }
 
@@ -403,7 +401,6 @@ export async function renderActiveWorkspaceRuntime(options: {
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
   workspaceSecretStore?: WorkspaceSecretStore;
-  catalogDatabase?: WorkspaceDatabase;
 }): Promise<ActiveRenderedWorkspaceRuntime> {
   // The persisted active state may reference host-side snapshot paths (written by another
   // process or installation). Read the active state directly and resolve the immutable snapshot
@@ -434,7 +431,6 @@ export async function renderActiveWorkspaceRuntime(options: {
     secretRoots: options.secretRoots,
     semanticRuntime: options.semanticRuntime,
     workspaceSecretStore: options.workspaceSecretStore,
-    catalogDatabase: options.catalogDatabase,
   });
   return {
     ...rendered,
@@ -484,7 +480,6 @@ export async function publishDeterministicRuntimeConfigLease(options: {
   secretRoots: readonly string[];
   semanticRuntime: SemanticRuntimeConfig;
   workspaceSecretStore?: WorkspaceSecretStore;
-  catalogDatabase?: WorkspaceDatabase;
 }): Promise<DeterministicRuntimeConfigLease> {
   const rendered = await renderActiveWorkspaceRuntime(options);
   const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
@@ -492,11 +487,7 @@ export async function publishDeterministicRuntimeConfigLease(options: {
   const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
   const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
   const configFingerprintValue = configFingerprint(renderedConfigObject);
-  const inputFingerprintValue = preprocessingInputFingerprint(
-    rendered.workspaceId,
-    rendered.workspaceRevision,
-    renderedConfigObject,
-  );
+  const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject);
   const identitySuffix = inputFingerprintValue.slice(7, 23);
 
   const preprocessingRoot = ensureTrustedDirectory(join(

backend/src/workspaces/runtime-renderer.ts

@@ -3,7 +3,6 @@ import { stringify } from "yaml";
 import { buildInstallationContract } from "./contracts.js";
 import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js";
 import type { ResolvedBinding, ResolvedEvidenceBinding, RuntimeBindings } from "./bindings.js";
-import { workspaceVectorCollections } from "./vector-collections.js";
 export type { RuntimeBindings } from "./bindings.js";
 
 export interface RuntimePaths {
@@ -11,8 +10,10 @@ export interface RuntimePaths {
   artifacts: string;
   indexes: string;
   memory: string;
-  /** Current backend-produced projection of the PostgreSQL Metadata Catalog. */
-  catalog_metadata_snapshot?: string;
+  /** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */
+  annotations_root?: string;
+  /** Immutable Catalog projection used as the exclusive runtime relationship source. */
+  effective_relationships?: string;
 }
 
 export interface RuntimeIdentity {
@@ -33,7 +34,6 @@ export interface RuntimeInstallationOverlay {
 export interface SemanticRuntimeConfig {
   internalQdrantUrl: string;
   internalEmbeddingUrl: string;
-  internalEmbeddingId?: string;
   internalEmbeddingModel: string;
   internalEmbeddingDimensions: number;
 }
@@ -41,7 +41,6 @@ export interface SemanticRuntimeConfig {
 export const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
   internalQdrantUrl: "http://qdrant:6333",
   internalEmbeddingUrl: "http://embedding:11434",
-  internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
   internalEmbeddingModel: "qwen3-embedding:0.6b",
   internalEmbeddingDimensions: 1024,
 };
@@ -203,16 +202,16 @@ function placeholderConnection(identity: { database: string; schema: string }):
 
 function requireSupportedDescriptor(workspace: unknown): void {
   if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Runtime renderer supports only workspace schema version 4");
+    throw new Error("Runtime renderer supports only workspace schema version 3");
   }
   const metadata = Reflect.get(workspace, "workspace");
   if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 4) {
-    throw new Error("Runtime renderer supports only workspace schema version 4");
+    || Reflect.get(metadata, "schema_version") !== 3) {
+    throw new Error("Runtime renderer supports only workspace schema version 3");
   }
 }
 
-/** Render the schema-v4 compatibility fields consumed by the current Python harness. */
+/** Render the schema-v3 compatibility fields consumed by the current Python harness. */
 export function renderRuntimeConfig(
   workspace: WorkspaceDescriptor,
   bindings: RuntimeBindings,
@@ -223,7 +222,6 @@ export function renderRuntimeConfig(
 ): string {
   requireSupportedDescriptor(workspace);
   const descriptor = validateWorkspaceDescriptor(workspace);
-  if (!descriptor.dwh) throw new Error("runtime configuration requires a Catalog database binding");
   const contract = buildInstallationContract(descriptor);
   const name = (role: "DWH" | "EVIDENCE", suffix: string) => {
     const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix);
@@ -243,7 +241,6 @@ export function renderRuntimeConfig(
   }
 
   const dwhIdentity = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
-  const vectorCollections = workspaceVectorCollections(descriptor.workspace.id);
   const database = bindings.dwh.transport === "postgres_direct"
     ? { ...directConnection(bindings.dwh, {
       host: name("DWH", "HOST"),
@@ -266,32 +263,16 @@ export function renderRuntimeConfig(
     ...(installation.profile === undefined ? {} : { profile: installation.profile }),
     language: descriptor.workspace.language,
     database,
-    semantic_index: {
-      vector_store: {
-        engine: "qdrant",
-        collections: vectorCollections,
-        dimensions: semanticRuntime.internalEmbeddingDimensions,
-        distance: "cosine",
-      },
-      embedding: {
-        provider: "ollama_internal",
-        id: semanticRuntime.internalEmbeddingId
-          ?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
-        model: semanticRuntime.internalEmbeddingModel,
-        dimensions: semanticRuntime.internalEmbeddingDimensions,
-      },
-    },
+    semantic_index: descriptor.semantic_index,
     resources: {
       vector: {
         engine: "qdrant",
         base_url: semanticRuntime.internalQdrantUrl,
-        collections: vectorCollections,
+        collection: descriptor.semantic_index.vector_store.collection,
       },
       embeddings: {
         provider: "ollama_internal",
         base_url: semanticRuntime.internalEmbeddingUrl,
-        id: semanticRuntime.internalEmbeddingId
-          ?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
         model: semanticRuntime.internalEmbeddingModel,
         dimensions: semanticRuntime.internalEmbeddingDimensions,
       },

backend/src/workspaces/schema.ts

@@ -35,7 +35,7 @@ export interface CanonicalDiagnostics {
 }
 
 interface WorkspaceMetadata {
-  schema_version: 4;
+  schema_version: 3;
   id: string;
   name: string;
   description?: string;
@@ -51,13 +51,31 @@ interface WorkspaceDwh {
   supported_transports: DwhTransport[];
 }
 
-interface WorkspaceBase {
+interface WorkspaceBase<TVectorStore> {
   workspace: WorkspaceMetadata;
-  /** Legacy connection block; current descriptors bind their database through PostgreSQL. */
-  dwh?: WorkspaceDwh;
+  dwh: WorkspaceDwh;
+  semantic_index: {
+    vector_store: TVectorStore;
+    embedding: {
+      provider: "ollama_internal";
+      model: "qwen3-embedding:0.6b";
+      dimensions: 1024;
+    };
+  };
+  llm_policy: {
+    default?: `${string}/${string}`;
+    allowed: `${string}/${string}`[];
+  };
   diagnostics?: Pick<CanonicalDiagnostics, "dwh_rest">;
 }
 
+interface QdrantVectorStore {
+  engine: "qdrant";
+  collection: string;
+  dimensions: 1024;
+  distance: "cosine";
+}
+
 export interface EvidencePolicy {
   max_chunk_chars: number;
   retain_published_generations: number;
@@ -102,12 +120,12 @@ export interface WorkspaceEvidence {
   policy: EvidencePolicy;
 }
 
-export interface WorkspaceV4 extends WorkspaceBase {
+export interface WorkspaceV3 extends WorkspaceBase<QdrantVectorStore> {
   evidence?: WorkspaceEvidence;
 }
 
-export type CanonicalWorkspace = WorkspaceV4;
-export type WorkspaceDescriptor = WorkspaceV4;
+export type CanonicalWorkspace = WorkspaceV3;
+export type WorkspaceDescriptor = WorkspaceV3;
 
 const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, {
   message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$",
@@ -117,6 +135,9 @@ const identifier = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, {
 });
 const port = z.number().int().min(1).max(65_535);
 const timeoutMs = z.number().int().positive();
+const modelReference = z.string().regex(/^[^/\s]+\/[^/\s]+$/, {
+  message: "model must use provider/model syntax",
+});
 
 function isOriginRelativeDiagnosticPath(value: string): boolean {
   return /^\/(?!\/)[^\\\u0000-\u001F\u007F?#]*$/.test(value) && !/%5c/i.test(value);
@@ -145,6 +166,21 @@ const dwhSchema = z.object({
   timeout_ms: timeoutMs.optional(),
   supported_transports: z.array(z.enum(DWH_TRANSPORTS)).min(1),
 }).strict();
+const internalEmbeddingSchema = z.object({
+  provider: z.literal("ollama_internal"),
+  model: z.literal("qwen3-embedding:0.6b"),
+  dimensions: z.literal(1024),
+}).strict();
+const qdrantVectorStoreSchema = z.object({
+  engine: z.literal("qdrant"),
+  collection: workspaceId,
+  dimensions: z.literal(1024),
+  distance: z.literal("cosine"),
+}).strict();
+const llmPolicySchema = z.object({
+  default: modelReference.optional(),
+  allowed: z.array(modelReference).min(1),
+}).strict();
 
 const positiveSafeInteger = z.number().int().safe().positive();
 const nonnegativeSafeInteger = z.number().int().safe().nonnegative();
@@ -329,9 +365,8 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert
 }
 
 function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
-  if (workspace.dwh) {
-    unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
-  }
+  unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
+  unique(workspace.llm_policy.allowed, context, ["llm_policy", "allowed"]);
 
   if (workspace.evidence?.source.type === "filesystem") {
     const expected = `${workspace.workspace.id}/evidence`;
@@ -344,8 +379,21 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
     }
   }
 
-  if (workspace.diagnostics?.dwh_rest
-    && !workspace.dwh?.supported_transports.includes("rest_api")) {
+  if (workspace.semantic_index.vector_store.dimensions !== workspace.semantic_index.embedding.dimensions) {
+    context.addIssue({
+      code: "custom",
+      path: ["semantic_index", "embedding", "dimensions"],
+      message: "embedding dimensions must match vector store dimensions",
+    });
+  }
+  if (workspace.llm_policy.default && !workspace.llm_policy.allowed.includes(workspace.llm_policy.default)) {
+    context.addIssue({
+      code: "custom",
+      path: ["llm_policy", "default"],
+      message: "LLM default must be included in the allowlist",
+    });
+  }
+  if (workspace.diagnostics?.dwh_rest && !workspace.dwh.supported_transports.includes("rest_api")) {
     context.addIssue({
       code: "custom",
       path: ["diagnostics", "dwh_rest"],
@@ -354,20 +402,25 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
   }
 }
 
-const WorkspaceV4Schema = z.object({
-  dwh: dwhSchema.optional(),
+const WorkspaceV3Schema = z.object({
+  dwh: dwhSchema,
+  llm_policy: llmPolicySchema,
   evidence: workspaceEvidenceSchema.optional(),
   diagnostics: z.object({
     dwh_rest: dwhRestDiagnostic.optional(),
   }).strict().optional(),
   workspace: z.object({
-    schema_version: z.literal(4), id: workspaceId, name: z.string().trim().min(1),
+    schema_version: z.literal(3), id: workspaceId, name: z.string().trim().min(1),
     description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]),
   }).strict(),
+  semantic_index: z.object({
+    vector_store: qdrantVectorStoreSchema,
+    embedding: internalEmbeddingSchema,
+  }).strict(),
 }).strict().superRefine(workspaceInvariants);
-const WorkspaceDescriptorSchema = WorkspaceV4Schema;
+const WorkspaceDescriptorSchema = WorkspaceV3Schema;
 
-function parseWorkspaceDocument(source: string): unknown {
+export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
   const documents = parseAllDocuments(source, { uniqueKeys: true });
   if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
   const document = documents[0];
@@ -375,48 +428,7 @@ function parseWorkspaceDocument(source: string): unknown {
     throw new Error(`Invalid workspace YAML: ${[...document.errors, ...document.warnings]
       .map((error) => error.message).join("; ")}`);
   }
-  return document.toJSON();
-}
-
-export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
-  const value = parseWorkspaceDocument(source);
-  assertAuthoredWorkspaceIsDatabaseFree(value);
-  return validateWorkspaceDescriptor(value);
-}
-
-/** Parses an ephemeral, generated core runtime descriptor that may contain a Catalog binding. */
-export function parseRuntimeWorkspaceYaml(source: string): WorkspaceDescriptor {
-  return validateWorkspaceDescriptor(parseWorkspaceDocument(source));
-}
-
-function assertAuthoredWorkspaceIsDatabaseFree(workspace: unknown): void {
-  if (workspace !== null && typeof workspace === "object"
-    && ("dwh" in workspace || "diagnostics" in workspace)) {
-    throw new Error(
-      "Workspace YAML must not contain database configuration; use the PostgreSQL Metadata Catalog",
-    );
-  }
-}
-
-/** Build a database-free v4 descriptor; legacy database/configuration fields are not carried over. */
-export function migrateWorkspaceV3Yaml(source: string): string {
-  const documents = parseAllDocuments(source, { uniqueKeys: true });
-  if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
-  const document = documents[0];
-  if (document.errors.length > 0 || document.warnings.length > 0) {
-    throw new Error("Invalid workspace YAML");
-  }
-  const value = document.toJSON() as Record<string, unknown>;
-  const metadata = value.workspace as Record<string, unknown> | undefined;
-  if (!metadata || metadata.schema_version !== 3) {
-    throw new Error("Workspace migration requires schema version 3");
-  }
-  metadata.schema_version = 4;
-  delete value.dwh;
-  delete value.diagnostics;
-  delete value.semantic_index;
-  delete value.llm_policy;
-  return serializeWorkspaceYaml(validateWorkspaceDescriptor(value));
+  return validateWorkspaceDescriptor(document.toJSON());
 }
 
 export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescriptor {
@@ -427,11 +439,11 @@ export function isCanonicalWorkspace(workspace: unknown): workspace is Canonical
   return WorkspaceDescriptorSchema.safeParse(workspace).success;
 }
 
-export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV4 {
+export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV3 {
   return WorkspaceDescriptorSchema.safeParse(workspace).success;
 }
 
-export function validateOperationalWorkspace(workspace: unknown): WorkspaceV4 {
+export function validateOperationalWorkspace(workspace: unknown): WorkspaceV3 {
   return validateWorkspaceDescriptor(workspace);
 }
 
@@ -449,7 +461,6 @@ export function resolveDiagnosticUrl(baseUrl: string, path: string): URL {
 
 export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string {
   const canonical = validateOperationalWorkspace(workspace);
-  assertAuthoredWorkspaceIsDatabaseFree(canonical);
   return stringify(canonical, { lineWidth: 0, sortMapEntries: true });
 }
 

backend/src/workspaces/secret-requirements.ts

@@ -101,8 +101,7 @@ function selectedTransport(
   descriptor: WorkspaceDescriptor,
   variables: readonly InstallationVariable[],
   env: NodeJS.ProcessEnv,
-): DwhTransport | undefined {
-  if (!descriptor.dwh) return undefined;
+): DwhTransport {
   const transportVariable = variables.find(({ role, suffix }) => role === "DWH" && suffix === "TRANSPORT");
   const value = transportVariable === undefined ? undefined : env[transportVariable.name];
   return isTransport(value) && descriptor.dwh.supported_transports.includes(value)
@@ -137,14 +136,11 @@ export function discoverWorkspaceSecretRequirements(
   const variables = buildInstallationContract(descriptor).variables;
   const transport = selectedTransport(descriptor, variables, env);
   const restHasNoAuthentication = transport === "rest_api" && descriptor.diagnostics?.dwh_rest?.auth === "none";
-  const requiredDwh = new Set(
-    transport === undefined || restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport],
-  );
+  const requiredDwh = new Set(restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport]);
 
   const requirements: WorkspaceSecretRequirement[] = [];
   for (const variable of variables) {
     if (variable.role === "DWH") {
-      if (transport === undefined) continue;
       if (variable.transports !== undefined && !variable.transports.includes(transport)) continue;
       const requirement = requirementFor(variable, requiredDwh.has(variable.suffix));
       if (requirement !== undefined && requirement.required) requirements.push(requirement);

backend/src/workspaces/types.ts

@@ -19,4 +19,4 @@ export type WorkspaceErrorCode =
   | "workspace_stale" | "git_unavailable" | "git_auth_failed" | "git_non_fast_forward"
   | "connector_unavailable" | "semantic_index_incompatible";
 
-export type { WorkspaceV4 } from "./schema.js";
+export type { WorkspaceV3 } from "./schema.js";

backend/src/workspaces/vector-collections.ts

@@ -1,20 +0,0 @@
-export interface WorkspaceVectorCollections {
-  reference: string;
-  memory: string;
-}
-
-/**
- * Physical Qdrant namespaces owned by one workspace.
- *
- * Reference data is replaceable preprocessing output. Memory is durable runtime
- * state and deliberately has a separate lifecycle.
- */
-export function workspaceVectorCollections(workspaceId: string): WorkspaceVectorCollections {
-  if (!/^[a-z][a-z0-9-]{2,62}$/.test(workspaceId)) {
-    throw new Error("workspace id is invalid");
-  }
-  return {
-    reference: `${workspaceId}-reference`,
-    memory: `${workspaceId}-memory`,
-  };
-}

backend/test/auth-pi-management-local.test.ts

@@ -5,12 +5,14 @@ import { createLocalAuthFixture } from "./auth-test-fixtures.js";
 function fakeService(): PiManagementService {
   return {
     status: vi.fn(async () => ({ ready: true })),
+    options: vi.fn(async () => ({ providers: [], models: [], reasoning: [], checkedAt: "2026-08-17T00:00:00.000Z" })),
+    configure: vi.fn(async (value) => ({ ...value, updatedAt: "2026-08-17T00:00:00.000Z" })),
     test: vi.fn(async () => ({ ready: true, checkedAt: "2026-08-17T00:00:00.000Z" })),
     logs: vi.fn(async () => ({ lines: [] })),
   };
 }
 
-test("a local HTTPS cookie session authorizes the Pi smoke check through an untrusted internal HTTP hop", async () => {
+test("a local HTTPS cookie session authorizes Pi writes through an untrusted internal HTTP hop", async () => {
   const service = fakeService();
   const fixture = await createLocalAuthFixture(
     { piManagement: service },
@@ -23,21 +25,31 @@ test("a local HTTPS cookie session authorizes the Pi smoke check through an untr
     expect(fixture.publicUrl).toBe("HTTPS://thothii.example.test");
 
     const proxyHeaders = fixture.sessionHeaders({ host: "127.0.0.1:8080" });
+    const configured = await fixture.app.inject({
+      method: "PUT",
+      url: "/pi-management/config",
+      headers: proxyHeaders,
+      payload: { provider: "zai", model: "glm-5.2", reasoning: "high" },
+    });
     const smoke = await fixture.app.inject({
       method: "POST",
       url: "/pi-management/test",
       headers: proxyHeaders,
     });
 
+    expect(configured.statusCode).toBe(200);
     expect(smoke.statusCode).toBe(200);
+    expect(service.configure).toHaveBeenCalledTimes(1);
     expect(service.test).toHaveBeenCalledTimes(1);
 
     fixture.resetDownstreamHits();
+    vi.mocked(service.configure).mockClear();
     vi.mocked(service.test).mockClear();
     const wrongOrigin = await fixture.app.inject({
-      method: "POST",
-      url: "/pi-management/test",
+      method: "PUT",
+      url: "/pi-management/config",
       headers: fixture.sessionHeaders({ host: "127.0.0.1:8080", origin: "https://evil.example" }),
+      payload: { provider: "zai", model: "glm-5.2", reasoning: "high" },
     });
     const wrongCsrf = await fixture.app.inject({
       method: "POST",
@@ -50,6 +62,7 @@ test("a local HTTPS cookie session authorizes the Pi smoke check through an untr
       expect(response.json()).toEqual({ code: "csrf_failed", error: "Request origin validation failed" });
     }
     expect(fixture.downstreamHits()).toBe(0);
+    expect(service.configure).not.toHaveBeenCalled();
     expect(service.test).not.toHaveBeenCalled();
   } finally {
     await fixture.close();

backend/test/auth-routes-local.test.ts

@@ -416,6 +416,19 @@ test("only two Argon2 verifications run concurrently and excess login attempts f
   expect((await second).statusCode).toBe(401);
 });
 
+test("the real native asynchronous Argon2 verifier holds two permits and releases them after completion", async () => {
+  const { app } = await createLocalApp();
+  const first = login(app, { password: `${password}!` });
+  const second = login(app, { password: `${password}!` });
+  await new Promise<void>((resolve) => setImmediate(resolve));
+
+  const excess = await login(app, { password: `${password}!` });
+  expect(excess.statusCode).toBe(429);
+  await expect(first).resolves.toMatchObject({ statusCode: 401 });
+  await expect(second).resolves.toMatchObject({ statusCode: 401 });
+  await expect(login(app, { password: `${password}!` })).resolves.toMatchObject({ statusCode: 401 });
+});
+
 test("a verifier failure is sanitized and releases its concurrency permit", async () => {
   let attempts = 0;
   const user = {

backend/test/catalog-databases-routes.test.ts

@@ -11,7 +11,6 @@ import type { ObservedSchemaSnapshot } from "../src/catalog/types.js";
 import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
 import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
-import type { WorkspaceDiagnoser } from "../src/routes/workspaces.js";
 
 const roots: string[] = [];
 afterEach(() => {
@@ -20,11 +19,16 @@ afterEach(() => {
 });
 
 const workspace: WorkspaceDescriptor = {
-  workspace: { schema_version: 4, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
+  workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
   dwh: {
     engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432,
     supported_transports: ["postgres_direct", "rest_api"],
   },
+  semantic_index: {
+    vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
+    embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+  },
+  llm_policy: { allowed: ["zai/glm-5.2"] },
   diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } },
 };
 const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
@@ -34,7 +38,6 @@ function setup(
   catalogDependencies: {
     catalogOperationCoordinator?: CatalogOperationCoordinator;
     catalogPostgresAccess?: CatalogPostgresAccess;
-    workspaceDiagnoser?: WorkspaceDiagnoser;
   } = {},
   workspaceDescriptor: WorkspaceDescriptor = workspace,
 ) {
@@ -58,7 +61,7 @@ function setup(
     workspaceRegistry: registry,
     workspaceSecretStore: secretStore,
     catalogRepository: repository,
-    workspaceDiagnoser: vi.fn(async () => ({ activatable: true, diagnostics: [] })),
+    workspaceDiagnoser: vi.fn(),
     ...catalogDependencies,
   });
   return { app, secretStore, repository };
@@ -97,23 +100,31 @@ const fleetSnapshot: ObservedSchemaSnapshot = {
   }],
 };
 
-test("lists every workspace and creates its Catalog database configuration", async () => {
+test("lists every YAML workspace and creates its one database configuration", async () => {
   const { app, secretStore } = setup();
   const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
   expect(initial.statusCode).toBe(200);
   expect(initial.json()).toMatchObject([{
     workspaceId: "psd-clinical",
     configured: false,
-    databaseName: "",
+    databaseName: "warehouse",
     workspaceRevision: { commit: revision.commit, blob: revision.blob },
     workspaceEvidence: { sourceType: null, state: "not_declared" },
-    runtimeBinding: null,
+    runtimeBinding: {
+      transport: "postgres_direct",
+      configurationState: "configuration_required",
+      sessionTransportSupported: true,
+    },
   }]);
 
+  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal");
+  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
+  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader");
+  vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
   secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" });
   const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" });
   expect(runtimeReady.json()).toMatchObject([{
-    runtimeBinding: null,
+    runtimeBinding: { configurationState: "ready", sessionTransportSupported: true },
   }]);
 
   const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
@@ -158,7 +169,7 @@ test("projects remote Evidence credential state without conflating catalog secre
   }]);
 });
 
-test("lists orphaned records and keeps the REST diagnostic path in the Catalog", async () => {
+test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
   const { app, repository } = setup();
   await repository.create({
     workspaceId: "removed-workspace",
@@ -178,7 +189,7 @@ test("lists orphaned records and keeps the REST diagnostic path in the Catalog",
     },
   });
   expect(created.statusCode).toBe(201);
-  expect(created.json()).toMatchObject({ binding: { restPath: "/client-controlled" } });
+  expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
 
   const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
   expect(rows).toEqual(expect.arrayContaining([
@@ -269,72 +280,6 @@ test("rejects a connection test while another catalog operation owns the databas
   }
 });
 
-test("workspace and database tests use the same current catalog database binding", async () => {
-  const connect = vi.fn(async () => ({
-    query: vi.fn(async () => ({
-      rows: [{ database: "warehouse", schema: "datawarehouse" }],
-    })),
-    end: vi.fn(async () => undefined),
-  }));
-  const diagnose: WorkspaceDiagnoser = vi.fn(async () => ({
-    activatable: true,
-    diagnostics: [{
-      level: "info",
-      code: "binding_ok",
-      message: "Installation bindings and diagnostics succeeded.",
-    }],
-  }));
-  const { app } = setup({
-    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
-    THT_WS_PSD_CLINICAL_DWH_HOST: "legacy-db.internal",
-    THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
-    THT_WS_PSD_CLINICAL_DWH_USER: "legacy-reader",
-  }, {
-    catalogPostgresAccess: { connect } as CatalogPostgresAccess,
-    workspaceDiagnoser: diagnose,
-  });
-  const created = (await app.inject({
-    method: "POST",
-    url: "/catalog/databases",
-    payload: {
-      ...direct,
-      binding: { ...direct.binding, host: "current-db.internal", username: "current-reader" },
-    },
-  })).json();
-
-  const databaseTest = await app.inject({
-    method: "POST",
-    url: `/catalog/databases/${created.id}/test`,
-    payload: { version: created.version },
-  });
-  const workspaceTest = await app.inject({
-    method: "POST",
-    url: "/workspaces/psd-clinical/test",
-    payload: {},
-  });
-
-  expect(databaseTest.statusCode).toBe(200);
-  expect(workspaceTest.statusCode).toBe(200);
-  expect(connect).toHaveBeenCalledTimes(2);
-  expect(connect.mock.calls.map(([database]) => database)).toEqual([
-    expect.objectContaining({
-      databaseName: "warehouse",
-      schema: "datawarehouse",
-      binding: expect.objectContaining({ host: "current-db.internal", username: "current-reader" }),
-    }),
-    expect.objectContaining({
-      databaseName: "warehouse",
-      schema: "datawarehouse",
-      binding: expect.objectContaining({ host: "current-db.internal", username: "current-reader" }),
-    }),
-  ]);
-  expect(diagnose).toHaveBeenCalledWith(
-    workspace,
-    expect.any(Object),
-    { writeProbe: false, skipDwh: true },
-  );
-});
-
 test("returns exact global and per-database fleet metrics", async () => {
   const { app, repository } = setup();
   const database = await repository.create(direct);

backend/test/catalog-description-generation-routes.test.ts

@@ -14,7 +14,6 @@ import {
   type ModelCompletionRequest,
 } from "../src/catalog/model-completer.js";
 import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
-import type { SensitivityValueSource } from "../src/catalog/sensitivity-classifier.js";
 import type {
   CatalogDatabaseClient,
   CatalogPostgresAccess,
@@ -25,7 +24,7 @@ import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
 
 const workspace: WorkspaceDescriptor = {
   workspace: {
-    schema_version: 4,
+    schema_version: 3,
     id: "psd-clinical",
     name: "Policlinico San Donato",
     language: "it",
@@ -37,6 +36,11 @@ const workspace: WorkspaceDescriptor = {
     port: 5432,
     supported_transports: ["postgres_direct"],
   },
+  semantic_index: {
+    vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
+    embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+  },
+  llm_policy: { allowed: ["zai/glm-5.2"] },
 };
 const revision: WorkspaceRevision = {
   id: "psd-clinical",
@@ -45,7 +49,7 @@ const revision: WorkspaceRevision = {
   snapshotPath: "/tmp/psd.yaml",
 };
 const configuredModel: ResolvedMetadataGenerationModel = {
-  id: "openai/gpt-4.1-mini",
+  id: "openai-mini",
   provider: "openai",
   model: "gpt-4.1-mini",
   apiKeyEnv: "OPENAI_API_KEY",
@@ -70,16 +74,6 @@ async function setup(
     sample: vi.fn(async () => []),
   },
   catalogPostgresAccess?: CatalogPostgresAccess,
-  sensitivityValueSource: SensitivityValueSource = {
-    scanTable: vi.fn(async (request, consume) => {
-      await consume(request.columns.map((column) => ({
-        columnId: column.id,
-        value: "ordinary",
-        characterLength: 8,
-      })));
-      return { kind: "complete", observedValues: 1 };
-    }),
-  },
 ) {
   const repository = new MemoryCatalogRepository();
   const database = await repository.create({
@@ -126,11 +120,10 @@ async function setup(
     catalogOperationCoordinator: operations,
     metadataGenerationModels: models(),
     modelCompleter,
-    sensitivityValueSource,
     ...(descriptionSourceSampler ? { descriptionSourceSampler } : {}),
     ...(catalogPostgresAccess ? { catalogPostgresAccess } : {}),
   });
-  return { app, repository, database, table, column, operations, sensitivityValueSource };
+  return { app, repository, database, table, column, operations };
 }
 
 async function waitForTerminalRun(app: ReturnType<typeof buildApp>, runId: string) {
@@ -148,15 +141,22 @@ async function waitForTerminalRun(app: ReturnType<typeof buildApp>, runId: strin
   throw new Error(`Description Generation Run ${runId} did not finish`);
 }
 
-test("assesses sensitive flags locally without persisting them or calling an LLM", async () => {
-  const modelCompleter: ModelCompleter = { complete: vi.fn(async () => "unused") };
+test("suggests sensitive flags from structural metadata without persisting them", async () => {
+  const modelCompleter = {
+    complete: vi.fn(async () => JSON.stringify({
+      suggestions: [{ columnId: expect.any(String), sensitive: true }],
+    })),
+  };
   const { app, repository, database, table, column } = await setup(modelCompleter);
+  modelCompleter.complete.mockResolvedValueOnce(JSON.stringify({
+    suggestions: [{ columnId: column.id, sensitive: true }],
+  }));
 
   try {
     const response = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { scope: "all" },
+      payload: { modelId: configuredModel.id, scope: "all" },
     });
 
     expect(response.statusCode).toBe(200);
@@ -165,14 +165,11 @@ test("assesses sensitive flags locally without persisting them or calling an LLM
       run: {
         databaseId: database.id,
         scope: "all",
-        engine: "local",
-        modelId: null,
-        policyVersion: "sensitivity-v4",
+        modelId: configuredModel.id,
         status: "completed",
         total: 1,
         suggestedSensitive: 1,
         suggestedNonSensitive: 0,
-        unknown: 0,
         errorSummary: null,
       },
       suggestions: [{
@@ -183,9 +180,6 @@ test("assesses sensitive flags locally without persisting them or calling an LLM
         version: column.version,
         currentSensitive: false,
         sensitive: true,
-        assessment: "sensitive",
-        evidence: [{ kind: "metadata", ruleId: "metadata.direct_identifier" }],
-        coverage: "metadata",
       }],
     });
     expect(await repository.getColumn(database.id, column.tableId, column.id))
@@ -218,69 +212,49 @@ test("assesses sensitive flags locally without persisting them or calling an LLM
         runId: responseBody.run.id,
         sequence: 1,
         level: "info",
-        message: "Local sensitivity analysis started.",
+        message: "Sensitive-field suggestion generation started.",
       },
       {
         runId: responseBody.run.id,
         sequence: 2,
         level: "info",
-        message: "Scanning source data: pass 1 of 3, table batch 1 of 1.",
+        message: "Classified 1 of 1 columns.",
       },
       {
         runId: responseBody.run.id,
         sequence: 3,
         level: "info",
-        message: "Scanning source data: pass 2 of 3, table batch 1 of 1.",
-      },
-      {
-        runId: responseBody.run.id,
-        sequence: 4,
-        level: "info",
-        message: "Scanning source data: pass 3 of 3, table batch 1 of 1.",
-      },
-      {
-        runId: responseBody.run.id,
-        sequence: 5,
-        level: "info",
-        message: "Assessed 1 of 1 columns locally.",
-      },
-      {
-        runId: responseBody.run.id,
-        sequence: 6,
-        level: "info",
-        message: "Local sensitivity analysis completed for 1 column.",
+        message: "Sensitive-field suggestion generation completed for 1 column.",
       },
     ]);
 
-    expect(modelCompleter.complete).not.toHaveBeenCalled();
+    const request = modelCompleter.complete.mock.calls[0]![0] as ModelCompletionRequest;
+    const prompt = request.messages.map((message) => message.content).join("\n");
+    expect(prompt).toContain("patients");
+    expect(prompt).toContain("birth_date");
+    expect(prompt).toContain("date");
+    expect(prompt).not.toContain("Patient date of birth");
+    expect(prompt).not.toContain("test-provider-secret");
   } finally {
     await app.close();
   }
 });
 
-test("does not impose a global HTTP deadline on sensitivity analysis", async () => {
-  const timeout = vi.spyOn(AbortSignal, "timeout");
-  const { app, repository, database } = await setup({ complete: vi.fn(async () => "unused") });
-
-  try {
-    const response = await app.inject({
-      method: "POST",
-      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { scope: "all" },
-    });
-
-    expect(response.statusCode).toBe(200);
-    expect(timeout).not.toHaveBeenCalled();
-    expect(await repository.listSensitivityAnalysisRuns()).toHaveLength(1);
-  } finally {
-    timeout.mockRestore();
-    await app.close();
-  }
-});
-
-test("limits sensitivity analysis to the selected tables or columns", async () => {
-  const modelCompleter: ModelCompleter = { complete: vi.fn(async () => "unused") };
-  const { app, repository, database, sensitivityValueSource } = await setup(modelCompleter);
+test("limits sensitive-data suggestions to the selected tables or columns", async () => {
+  const modelCompleter: ModelCompleter = {
+    complete: vi.fn(async (request) => {
+      const payload = JSON.parse(request.messages.find((message) => message.role === "user")!.content) as {
+        columns: Array<{ columnId: string; column: string }>;
+      };
+      return JSON.stringify({
+        suggestions: payload.columns.map((column) => ({
+          columnId: column.columnId,
+          sensitive: column.column.includes("name") || column.column.includes("note"),
+        })),
+      });
+    }),
+  };
+  const { app, repository, database } = await setup(modelCompleter);
   await repository.applySchemaSync(database.id, database.version, "all", [], {
     schemaVersion: 1,
     capabilities: { tables: "available", columns: "available", relationships: "available" },
@@ -312,6 +286,7 @@ test("limits sensitivity analysis to the selected tables or columns", async () =
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
+        modelId: configuredModel.id,
         scope: "selected_tables",
         targetIds: [visits.id, patients.id],
       },
@@ -331,6 +306,7 @@ test("limits sensitivity analysis to the selected tables or columns", async () =
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
+        modelId: configuredModel.id,
         scope: "selected_columns",
         targetIds: [clinicalNote.id, status.id],
       },
@@ -342,41 +318,44 @@ test("limits sensitivity analysis to the selected tables or columns", async () =
       expect.objectContaining({ tableId: visits.id, columnId: clinicalNote.id, sensitive: true }),
     ]));
 
-    const scannedColumnIds = vi.mocked(sensitivityValueSource.scanTable).mock.calls.flatMap(
-      ([request]) => request.columns.map((column) => column.id),
+    const prompts = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => (
+      JSON.parse(request.messages.find((message) => message.role === "user")!.content) as {
+        columns: Array<{ columnId: string }>;
+      }
+    ));
+    expect(prompts[0]!.columns.map((column) => column.columnId).sort()).toEqual(
+      [...patientColumns, ...visitColumns].map((column) => column.id).sort(),
+    );
+    expect(prompts[0]!.columns.map((column) => column.columnId)).not.toContain(billingColumns[0]!.id);
+    expect(prompts[1]!.columns.map((column) => column.columnId).sort()).toEqual(
+      [status.id, clinicalNote.id].sort(),
     );
-    expect(scannedColumnIds).toEqual([status.id, status.id]);
-    expect(scannedColumnIds).not.toContain(patientColumns.find(
-      (column) => column.name === "patient_name",
-    )!.id);
-    expect(scannedColumnIds).not.toContain(clinicalNote.id);
-    expect(scannedColumnIds).not.toContain(billingColumns[0]!.id);
-    expect(modelCompleter.complete).not.toHaveBeenCalled();
   } finally {
     await app.close();
   }
 });
 
-test("explains invalid sensitivity-analysis selections without reading source values", async () => {
+test("explains invalid sensitive-data suggestion selections without calling the model", async () => {
   const modelCompleter: ModelCompleter = { complete: vi.fn(async () => "unused") };
-  const { app, database, table, sensitivityValueSource } = await setup(modelCompleter);
+  const { app, database, table } = await setup(modelCompleter);
 
   try {
     const empty = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
-      payload: { scope: "selected_tables", targetIds: [] },
+      payload: { modelId: configuredModel.id, scope: "selected_tables", targetIds: [] },
     });
     expect(empty.statusCode).toBe(400);
     expect(empty.json()).toEqual({
       code: "sensitive_data_suggestion_request_invalid",
-      message: "Choose a database, one or more tables, or one or more columns to assess.",
+      message: "Choose a database, one or more tables, or one or more columns to classify.",
     });
 
     const duplicate = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
+        modelId: configuredModel.id,
         scope: "selected_tables",
         targetIds: [table.id, table.id],
       },
@@ -391,6 +370,7 @@ test("explains invalid sensitivity-analysis selections without reading source va
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
+        modelId: configuredModel.id,
         scope: "selected_tables",
         targetIds: ["00000000-0000-4000-8000-000000000001"],
       },
@@ -405,6 +385,7 @@ test("explains invalid sensitivity-analysis selections without reading source va
       method: "POST",
       url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
       payload: {
+        modelId: configuredModel.id,
         scope: "selected_columns",
         targetIds: ["00000000-0000-4000-8000-000000000002"],
       },
@@ -415,7 +396,205 @@ test("explains invalid sensitivity-analysis selections without reading source va
       message: "One or more selected Catalog Columns were not found in this database.",
     });
     expect(modelCompleter.complete).not.toHaveBeenCalled();
-    expect(sensitivityValueSource.scanTable).not.toHaveBeenCalled();
+  } finally {
+    await app.close();
+  }
+});
+
+test("batches sensitive-data suggestions for schemas larger than one helper message", async () => {
+  const maxHelperMessageBytes = 64 * 1024;
+  const seenColumnIds: string[] = [];
+  const modelCompleter: ModelCompleter = {
+    complete: vi.fn(async (request) => {
+      const userMessage = request.messages.find((message) => message.role === "user")!;
+      expect(Buffer.byteLength(userMessage.content, "utf8")).toBeLessThanOrEqual(maxHelperMessageBytes);
+      const payload = JSON.parse(userMessage.content) as {
+        columns: Array<{ columnId: string; column: string }>;
+      };
+      expect(payload.columns.length).toBeLessThanOrEqual(10);
+      seenColumnIds.push(...payload.columns.map((column) => column.columnId));
+      return JSON.stringify({
+        suggestions: payload.columns.map((column) => ({
+          columnId: column.columnId,
+          sensitive: column.column.endsWith("_private"),
+        })),
+      });
+    }),
+  };
+  const { app, repository, database } = await setup(modelCompleter);
+  const columnCount = 900;
+  await repository.applySchemaSync(database.id, database.version, "all", [], {
+    schemaVersion: 1,
+    capabilities: { tables: "available", columns: "available", relationships: "available" },
+    tables: [{ name: "wide_table", sourceComment: null }],
+    columns: Array.from({ length: columnCount }, (_, index) => ({
+      tableName: "wide_table",
+      name: `field_${index.toString().padStart(4, "0")}${index % 10 === 0 ? "_private" : ""}`,
+      ordinalPosition: index + 1,
+      dataType: "character varying(255)",
+      isNullable: true,
+      defaultExpression: null,
+      primaryKeyPosition: null,
+      sourceComment: null,
+    })),
+    relationships: [],
+  });
+  const wideTable = (await repository.listTables(database.id)).find((table) => table.name === "wide_table")!;
+  const expectedColumnIds = (await repository.listColumns(database.id, wideTable.id)).map((column) => column.id);
+
+  try {
+    const response = await app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
+      payload: { modelId: configuredModel.id, scope: "all" },
+    });
+
+    expect(response.statusCode).toBe(200);
+    const suggestions = response.json().suggestions as Array<{
+      columnName: string;
+      currentSensitive: boolean;
+      sensitive: boolean;
+    }>;
+    expect(suggestions).toHaveLength(columnCount);
+    expect(suggestions).toEqual(expect.arrayContaining([
+      expect.objectContaining({ columnName: "field_0000_private", currentSensitive: false, sensitive: true }),
+      expect.objectContaining({ columnName: "field_0001", currentSensitive: false, sensitive: false }),
+    ]));
+    expect(vi.mocked(modelCompleter.complete).mock.calls.length).toBeGreaterThan(1);
+    expect(seenColumnIds.slice().sort()).toEqual(expectedColumnIds.slice().sort());
+    expect(new Set(seenColumnIds).size).toBe(columnCount);
+  } finally {
+    await app.close();
+  }
+});
+
+test("retries one invalid sensitive-data classification before returning the review draft", async () => {
+  const modelCompleter: ModelCompleter = {
+    complete: vi.fn(async () => "unused"),
+  };
+  const { app, database, column } = await setup(modelCompleter);
+  vi.mocked(modelCompleter.complete)
+    .mockResolvedValueOnce("not-json")
+    .mockResolvedValueOnce(JSON.stringify({
+      suggestions: [{ columnId: column.id, sensitive: true }],
+    }));
+
+  try {
+    const response = await app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
+      payload: { modelId: configuredModel.id, scope: "all" },
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json().suggestions).toEqual([
+      expect.objectContaining({ columnId: column.id, sensitive: true }),
+    ]);
+    expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
+  } finally {
+    await app.close();
+  }
+});
+
+test.each(["malformed", "incomplete", "duplicate"] as const)(
+  "fails safely when sensitive-data suggestions are %s",
+  async (kind) => {
+    const modelCompleter: ModelCompleter = {
+      complete: vi.fn(async () => "unused"),
+    };
+    const { app, repository, database, column } = await setup(modelCompleter);
+    const rawResponse = kind === "malformed"
+      ? "RAW_PROVIDER_RESPONSE_DO_NOT_EXPOSE_{"
+      : kind === "incomplete"
+        ? JSON.stringify({ suggestions: [] })
+        : JSON.stringify({
+          suggestions: [
+            { columnId: column.id, sensitive: true },
+            { columnId: column.id, sensitive: true },
+          ],
+        });
+    vi.mocked(modelCompleter.complete).mockResolvedValueOnce(rawResponse);
+
+    try {
+      const response = await app.inject({
+        method: "POST",
+        url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
+        payload: { modelId: configuredModel.id, scope: "all" },
+      });
+
+      expect(response.statusCode).toBe(502);
+      expect(response.json()).toEqual({
+        code: "sensitive_data_suggestion_invalid_response",
+        message: "The LLM returned an incomplete or invalid classification. No suggestions were applied.",
+      });
+      expect(response.body).not.toContain(rawResponse);
+      expect(await repository.getColumn(database.id, column.tableId, column.id))
+        .toMatchObject({ sensitive: false });
+    } finally {
+      await app.close();
+    }
+  },
+);
+
+test("explains a sensitive-data suggestion provider failure without exposing provider details", async () => {
+  const modelCompleter: ModelCompleter = {
+    complete: vi.fn(async () => {
+      throw new ModelCompletionProviderError();
+    }),
+  };
+  const { app, repository, database, column } = await setup(modelCompleter);
+
+  try {
+    const response = await app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
+      payload: { modelId: configuredModel.id, scope: "all" },
+    });
+
+    expect(response.statusCode).toBe(502);
+    expect(response.json()).toEqual({
+      code: "sensitive_data_suggestion_provider_unavailable",
+      message: "The selected LLM service could not complete the request. No suggestions were applied.",
+    });
+    expect(response.body).not.toContain("model completion failed");
+    expect(await repository.getColumn(database.id, column.tableId, column.id))
+      .toMatchObject({ sensitive: false });
+
+    const history = await app.inject({
+      method: "GET",
+      url: "/catalog/sensitive-data-suggestion-runs",
+    });
+    expect(history.statusCode).toBe(200);
+    const [failedRun] = history.json();
+    expect(failedRun).toMatchObject({
+      databaseId: database.id,
+      status: "failed",
+      total: 1,
+      suggestedSensitive: 0,
+      suggestedNonSensitive: 0,
+      errorSummary: "Sensitive-field suggestion generation failed.",
+    });
+
+    const events = await app.inject({
+      method: "GET",
+      url: `/catalog/sensitive-data-suggestion-runs/${failedRun.id}/events-list`,
+    });
+    expect(events.statusCode).toBe(200);
+    expect(events.json()).toMatchObject([
+      {
+        runId: failedRun.id,
+        sequence: 1,
+        level: "info",
+        message: "Sensitive-field suggestion generation started.",
+      },
+      {
+        runId: failedRun.id,
+        sequence: 2,
+        level: "error",
+        message: "Sensitive-field suggestion generation failed.",
+      },
+    ]);
+    expect(events.body).not.toContain("model completion failed");
   } finally {
     await app.close();
   }
@@ -526,7 +705,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
     expect(completionRequest.messages[0]?.content).toContain('{"results":[');
     expect(completionRequest.messages[1]?.content).toContain(`"targetId":"${column.id}"`);
     expect(completionRequest.messages[1]?.content).not.toMatch(/source rows|samples|example values/i);
-    expect(start.body).not.toMatch(/test-provider-secret|Catalog metadata/);
+    expect(start.body).not.toMatch(/test-provider-secret|gpt-4\.1|openai\/gpt|Catalog metadata/);
 
     resolveCompletion(`\`\`\`json\n${JSON.stringify({
       results: [{
@@ -577,7 +756,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
       expect.objectContaining({
         sequence: 3,
         level: "info",
-        message: 'Generated description for Column "patients.birth_date".',
+        message: `Generated description for Catalog Column ${column.id}.`,
       }),
       expect.objectContaining({ sequence: 4, level: "info", message: "Description generation completed." }),
     ]);
@@ -934,10 +1113,7 @@ test("generates selected Catalog Columns in caller order through sequential batc
     const events = await repository.listDescriptionGenerationEvents(run.id);
     expect(events.map((event) => event.sequence)).toEqual(Array.from({ length: 14 }, (_, index) => index + 1));
     expect(events.slice(2, -1).map((event) => event.message)).toEqual(
-      orderedIds.map((targetId) => {
-        const target = columns.find((column) => column.id === targetId)!;
-        return `Generated description for Column "patients.${target.name}".`;
-      }),
+      orderedIds.map((targetId) => `Generated description for Catalog Column ${targetId}.`),
     );
   } finally {
     pending[0]!.resolve(responseFor(orderedIds.slice(0, 10), 0));
@@ -1806,9 +1982,7 @@ test("retains completed batch writes when a later batch response is malformed",
     });
     const events = await repository.listDescriptionGenerationEvents(run.id);
     expect(events.slice(2, 12).map((event) => event.message)).toEqual(
-      originalColumns.slice(0, 10).map(
-        (target) => `Generated description for Column "patients.${target.name}".`,
-      ),
+      orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
     );
     expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
       expect.objectContaining({
@@ -1817,7 +1991,7 @@ test("retains completed batch writes when a later batch response is malformed",
     );
     expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
       level: "error",
-      message: `The model response did not match the required schema. Affected target: Column "patients.${originalColumns[10]!.name}".`,
+      message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
     }));
   } finally {
     await app.close();
@@ -2306,7 +2480,7 @@ test("generates selected Catalog Tables with structural column context and local
     expect((await repository.listDescriptionGenerationEvents(run.id)).map((event) => event.message)).toEqual([
       "Description generation queued.",
       "Description generation started.",
-      'Stored non-generatable result for Table "patients".',
+      `Stored non-generatable result for Catalog Table ${table.id}.`,
       "Description generation completed.",
     ]);
   } finally {
@@ -2450,7 +2624,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
     expect(`${JSON.stringify(run)}${events.body}`).not.toContain(sensitiveDiagnostic);
     expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
       level: "error",
-      message: 'The model provider request failed. Affected target: Column "patients.birth_date".',
+      message: `The model provider request failed. Affected Catalog Column target: ${column.id}.`,
     }));
   } finally {
     await app.close();
@@ -2541,7 +2715,7 @@ test.each([
     expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
       expect.objectContaining({
         level: "error",
-        message: `${failureMessage} Affected targets: Column "patients.first_column", Column "patients.second_column".`,
+        message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
       }),
     );
   } finally {
@@ -2735,7 +2909,7 @@ test("validates selected targets and resolves every requested target before laun
     const unknownModel = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai/unknown-model", scope: "selected_columns", targetIds: [column.id] },
+      payload: { modelId: "unknown-model", scope: "selected_columns", targetIds: [column.id] },
     });
     expect(unknownModel.statusCode).toBe(409);
     expect(unknownModel.json().code).toBe("metadata_generation_model_unavailable");

backend/test/catalog-description-generation.integration.test.ts

@@ -13,12 +13,7 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
 import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
 import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
 import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
-import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
 import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
-import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
-import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
-import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
-import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
 import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
 import { loadConfig } from "../src/config.js";
 import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
@@ -54,12 +49,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     await upSensitiveDataFlag(db);
     await upDescriptionGeneration(db);
     await upSensitiveSuggestionRuns(db);
-    await upLogicalRelationships(db);
     await upAiTokenUsage(db);
-    await upCanonicalModelIds(db);
-    await upLocalSensitivityAnalysis(db);
-    await upSensitivityReason(db);
-    await upCatalogPreprocessingState(db);
     const repository = new KyselyCatalogRepository(db);
     const database = await repository.create({
       workspaceId: "psd-clinical",
@@ -168,9 +158,9 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
       }),
     };
     const models: MetadataGenerationModels = {
-      catalog: () => ({ models: [{ id: "openai/gpt-4.1-mini", label: "OpenAI Mini" }], default: "openai/gpt-4.1-mini" }),
+      catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
       resolve: () => ({
-        id: "openai/gpt-4.1-mini",
+        id: "openai-mini",
         provider: "openai",
         model: "gpt-4.1-mini",
         apiKeyEnv: "OPENAI_API_KEY",
@@ -215,12 +205,12 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
       payload: {
-        modelId: "openai/gpt-4.1-mini",
+        modelId: "openai-mini",
         scope: "selected_columns",
         targetIds: [status.id, birthDate.id],
       },
     });
-    expect(successfulStart.statusCode, successfulStart.body).toBe(202);
+    expect(successfulStart.statusCode).toBe(202);
     expect(await terminalRun(app, successfulStart.json().id)).toMatchObject({
       status: "completed",
       total: 2,
@@ -243,7 +233,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     const tableStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai/gpt-4.1-mini", scope: "selected_tables", targetIds: [table.id] },
+      payload: { modelId: "openai-mini", scope: "selected_tables", targetIds: [table.id] },
     });
     expect(tableStart.statusCode).toBe(202);
     expect(await terminalRun(app, tableStart.json().id)).toMatchObject({
@@ -263,7 +253,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     const failedStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai/gpt-4.1-mini", scope: "selected_columns", targetIds: [status.id] },
+      payload: { modelId: "openai-mini", scope: "selected_columns", targetIds: [status.id] },
     });
     expect(failedStart.statusCode).toBe(202);
     const failedRun = await terminalRun(app, failedStart.json().id);
@@ -286,14 +276,14 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     expect(events.statusCode).toBe(200);
     expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
       level: "error",
-      message: 'The model provider request failed. Affected target: Column "patients.status".',
+      message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
     }));
     expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
 
     const allStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai/gpt-4.1-mini", scope: "all" },
+      payload: { modelId: "openai-mini", scope: "all" },
     });
     expect(allStart.statusCode).toBe(202);
     const allRun = await terminalRun(app, allStart.json().id);
@@ -337,7 +327,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
     const missingStart = await app.inject({
       method: "POST",
       url: `/catalog/databases/${database.id}/description-generation-runs`,
-      payload: { modelId: "openai/gpt-4.1-mini", scope: "missing" },
+      payload: { modelId: "openai-mini", scope: "missing" },
     });
     expect(missingStart.statusCode).toBe(202);
     expect(await terminalRun(app, missingStart.json().id)).toMatchObject({

backend/test/catalog-local-ner-detector.test.ts

@@ -1,131 +0,0 @@
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterEach, expect, test, vi } from "vitest";
-import { PythonLocalNerDetector } from "../src/catalog/local-ner-detector.js";
-
-const roots: string[] = [];
-
-afterEach(() => {
-  vi.unstubAllEnvs();
-  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
-});
-
-test("keeps a CPU-only local worker warm and returns sanitized evidence", async () => {
-  vi.stubEnv("THT_MODEL_API_KEY", "must-not-reach-worker");
-  const root = mkdtempSync(join(tmpdir(), "thothii-local-ner-"));
-  roots.push(root);
-  const helper = join(root, "fake_ner_worker.py");
-  writeFileSync(helper, `
-import json
-import os
-import pathlib
-import sys
-
-root = pathlib.Path.cwd()
-root.joinpath("runtime.json").write_text(json.dumps({
-    "argv": sys.argv,
-    "cuda": os.environ.get("CUDA_VISIBLE_DEVICES"),
-    "hip": os.environ.get("HIP_VISIBLE_DEVICES"),
-    "offline": os.environ.get("HF_HUB_OFFLINE"),
-    "inherited_secret": os.environ.get("THT_MODEL_API_KEY"),
-    "pid": os.getpid(),
-}), encoding="utf-8")
-print(json.dumps({"ready": True}), flush=True)
-for line in sys.stdin:
-    request = json.loads(line)
-    root.joinpath("request.json").write_text(json.dumps(request), encoding="utf-8")
-    print(json.dumps({
-        "id": request["id"],
-        "ok": True,
-        "evidence": [{
-            "columnId": request["candidates"][0]["columnId"],
-            "label": "person",
-            "confidence": 0.93,
-        }],
-    }), flush=True)
-`, "utf8");
-  const detector = new PythonLocalNerDetector({
-    pythonExecutable: "python3",
-    workerScript: helper,
-    modelPath: join(root, "pinned-model"),
-    cwd: root,
-    threads: 2,
-    startupTimeoutMs: 5_000,
-  });
-  const candidate = {
-    columnId: "33333333-3333-4333-8333-333333333333",
-    text: "Dimesso Mario Rossi",
-  };
-
-  try {
-    expect(detector.isReady()).toBe(false);
-    await detector.warmup();
-    expect(detector.isReady()).toBe(true);
-    expect(existsSync(join(root, "request.json"))).toBe(false);
-
-    await expect(detector.detect(
-      [candidate],
-      new AbortController().signal,
-      Date.now() + 5_000,
-    )).resolves.toEqual([{
-      columnId: candidate.columnId,
-      label: "person",
-      confidence: 0.93,
-    }]);
-    const firstRuntime = JSON.parse(readFileSync(join(root, "runtime.json"), "utf8"));
-    expect(firstRuntime).toMatchObject({
-      cuda: "",
-      hip: "",
-      offline: "1",
-      inherited_secret: null,
-    });
-    expect(JSON.stringify(firstRuntime.argv)).not.toContain(candidate.text);
-    expect(JSON.parse(readFileSync(join(root, "request.json"), "utf8")).candidates).toEqual([candidate]);
-
-    await detector.detect([candidate], new AbortController().signal, Date.now() + 5_000);
-    const secondRuntime = JSON.parse(readFileSync(join(root, "runtime.json"), "utf8"));
-    expect(secondRuntime.pid).toBe(firstRuntime.pid);
-  } finally {
-    await detector.close();
-  }
-});
-
-test("bounds worker startup by the caller deadline", async () => {
-  const root = mkdtempSync(join(tmpdir(), "thothii-local-ner-deadline-"));
-  roots.push(root);
-  const helper = join(root, "slow_ner_worker.py");
-  writeFileSync(helper, `
-import json
-import sys
-import time
-
-time.sleep(2)
-print(json.dumps({"ready": True}), flush=True)
-for line in sys.stdin:
-    request = json.loads(line)
-    print(json.dumps({"id": request["id"], "ok": True, "evidence": []}), flush=True)
-`, "utf8");
-  const detector = new PythonLocalNerDetector({
-    pythonExecutable: "python3",
-    workerScript: helper,
-    modelPath: join(root, "pinned-model"),
-    cwd: root,
-    startupTimeoutMs: 5_000,
-  });
-  const startedAt = Date.now();
-
-  try {
-    await expect(detector.detect(
-      [{
-        columnId: "33333333-3333-4333-8333-333333333333",
-        text: "Dimesso Mario Rossi",
-      }],
-      new AbortController().signal,
-      startedAt + 50,
-    )).rejects.toThrow("local NER is unavailable");
-    expect(Date.now() - startedAt).toBeLessThan(1_000);
-  } finally {
-    await detector.close();
-  }
-});

backend/test/catalog-repository.integration.test.ts

@@ -1,5 +1,4 @@
 import { spawnSync } from "node:child_process";
-import { randomUUID } from "node:crypto";
 import { PostgreSqlContainer } from "@testcontainers/postgresql";
 import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
 import { Pool } from "pg";
@@ -15,10 +14,6 @@ import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensiti
 import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
 import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
 import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
-import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
-import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
-import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
-import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
 
 const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
 
@@ -37,43 +32,6 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
     await upDescriptionGeneration(db);
     await upSensitiveSuggestionRuns(db);
     await upAiTokenUsage(db);
-    const historicalDatabaseId = randomUUID();
-    await db.insertInto("workspaceDatabases").values({
-      id: historicalDatabaseId,
-      workspaceId: "migration-history",
-      engine: "postgres",
-      databaseName: "warehouse",
-      schemaName: "public",
-    }).execute();
-    await db.insertInto("descriptionGenerationRuns").values({
-      id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
-      modelId: "openai-mini", language: "en", status: "completed", total: 1,
-      processed: 1, generated: 1,
-    }).execute();
-    const historicalSuggestionRunId = randomUUID();
-    await db.insertInto("sensitiveDataSuggestionRuns").values({
-      id: historicalSuggestionRunId, databaseId: historicalDatabaseId, scope: "all",
-      modelId: "openai-mini", status: "completed", total: 1,
-      suggestedSensitive: 1,
-    }).execute();
-    await upCanonicalModelIds(db);
-    await upLocalSensitivityAnalysis(db);
-    await upSensitivityReason(db);
-    await upCatalogPreprocessingState(db);
-    await expect(db.selectFrom("sensitiveDataSuggestionRuns")
-      .select(["engine", "modelId", "policyVersion", "unknown"])
-      .where("id", "=", historicalSuggestionRunId)
-      .executeTakeFirstOrThrow()).resolves.toMatchObject({
-      engine: "llm",
-      modelId: "openai-mini",
-      policyVersion: null,
-      unknown: 0,
-    });
-    await expect(db.insertInto("descriptionGenerationRuns").values({
-      id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
-      modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1,
-      processed: 1, generated: 1,
-    }).execute()).resolves.toBeDefined();
     await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
     await upRuntimeSequencePrivileges(db);
     const sequencePrivilege = await sql<{ allowed: boolean }>`
@@ -146,11 +104,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
       patientName.description,
       patientName.generatedDescription,
       true,
-      "Local assessment matched content rule pii.person_name.",
-    )).toMatchObject({
-      sensitive: true,
-      sensitivityReason: "Local assessment matched content rule pii.person_name.",
-    });
+    )).toMatchObject({ sensitive: true });
     expect(await repository.getCatalogMetrics(created.id)).toEqual({
       scope: "database",
       databaseId: created.id,
@@ -195,7 +149,6 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
     )).toMatchObject({ updated: 1 });
     expect(await repository.getColumn(created.id, patients.id, patientName.id)).toMatchObject({
       sensitive: true,
-      sensitivityReason: "Local assessment matched content rule pii.person_name.",
       sourceComment: "Sensitive patient name",
     });
 
@@ -234,77 +187,6 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
     expect(await repository.listSyncRuns(created.id)).toEqual([
       expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
     ]);
-
-    const lockedDatabase = await repository.create({
-      workspaceId: "preprocessing-lock",
-      engine: "postgres",
-      databaseName: "warehouse",
-      schema: "public",
-      binding: {
-        transport: "postgres_direct", host: "lock.internal", port: 5432, username: "reader",
-      },
-    });
-    await repository.applySchemaSync(
-      lockedDatabase.id,
-      lockedDatabase.version,
-      "all",
-      [],
-      {
-        schemaVersion: 1,
-        capabilities: { tables: "available", columns: "available", relationships: "available" },
-        tables: [], columns: [], relationships: [],
-      },
-    );
-    const beforePreprocessing = (await repository.get(lockedDatabase.id))!;
-    const preprocessing = await repository.beginPreprocessing(
-      "preprocessing-lock",
-      "sha256:" + "1".repeat(64),
-    );
-    expect(preprocessing).toMatchObject({ kind: "started" });
-    await expect(db.insertInto("catalogTables").values({
-      id: randomUUID(),
-      databaseId: lockedDatabase.id,
-      name: "blocked_write",
-      sourceComment: null,
-      description: null,
-      generatedDescription: null,
-    }).execute()).rejects.toThrow("catalog preprocessing is running");
-    await expect(repository.createDescriptionGenerationRun(
-      lockedDatabase.id,
-      "all",
-      "openai/gpt-5-mini",
-      "en",
-      0,
-    )).rejects.toThrow("catalog preprocessing is running");
-    await repository.recordTest(lockedDatabase.id, lockedDatabase.version, {
-      connectionStatus: "reachable",
-      testedVersion: lockedDatabase.version,
-      lastTestedAt: "2026-01-01T00:00:00Z",
-    });
-    expect((await repository.get(lockedDatabase.id))?.metadataContentRevision)
-      .toBe(beforePreprocessing.metadataContentRevision);
-    await expect(repository.finishPreprocessing(
-      "preprocessing-lock",
-      beforePreprocessing.metadataContentRevision,
-      "sha256:" + "1".repeat(64),
-      { status: "succeeded" },
-    )).resolves.toMatchObject({
-      preprocessingStatus: "succeeded",
-      preprocessedMetadataRevision: beforePreprocessing.metadataContentRevision,
-    });
-    await db.insertInto("catalogTables").values({
-      id: randomUUID(),
-      databaseId: lockedDatabase.id,
-      name: "allowed_after_preprocessing",
-      sourceComment: null,
-      description: null,
-      generatedDescription: null,
-    }).execute();
-    await expect(repository.get(lockedDatabase.id)).resolves.toMatchObject({
-      preprocessingStatus: "failed",
-      metadataContentRevision: beforePreprocessing.metadataContentRevision + 1,
-    });
-    expect(await repository.delete(lockedDatabase.id, lockedDatabase.version)).toBe(true);
     expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
     expect(await repository.delete(created.id, 2)).toBe(true);
     expect(await repository.list()).toEqual([]);
@@ -361,33 +243,10 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
     };
     await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
     const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
-    const context = (await repository.getLogicalRelationshipContext(database.id))!;
-    const source = context.endpoints.find((endpoint) => (
-      endpoint.tableName === "visits" && endpoint.columnName === "id"
-    ))!;
-    const target = context.endpoints.find((endpoint) => (
-      endpoint.tableName === "patients" && endpoint.columnName === "id"
-    ))!;
-    const generatedCandidate = { sourceColumnId: source.columnId, targetColumnId: target.columnId };
 
-    await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate]))
-      .resolves.toBe(1);
-    await expect(repository.getCatalogMetrics(database.id))
-      .resolves.toMatchObject({ relationships: 2 });
-    expect(await repository.deleteDatabaseMetadata([database.id], "relationships"))
-      .toEqual({ tables: 0, columns: 0, relationships: 2 });
-    expect(await repository.listRelationships(database.id)).toEqual([]);
-    expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
-    await expect(repository.getCatalogMetrics(database.id))
-      .resolves.toMatchObject({ relationships: 0 });
-
-    await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
-    await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate]))
-      .resolves.toBe(1);
     expect(await repository.deleteTableMetadata(database.id, [patients.id], "relationships"))
-      .toEqual({ tables: 0, columns: 0, relationships: 2 });
+      .toEqual({ tables: 0, columns: 0, relationships: 1 });
     expect(await repository.listRelationships(database.id)).toEqual([]);
-    expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
     expect(await repository.listColumns(database.id, patients.id)).toHaveLength(2);
     expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
 
@@ -401,24 +260,13 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
 
     await repository.applySchemaSync(database.id, database.version, "columns", [patients.id], snapshot);
     await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
-    const refreshedContext = (await repository.getLogicalRelationshipContext(database.id))!;
-    const refreshedSource = refreshedContext.endpoints.find((endpoint) => (
-      endpoint.tableName === "visits" && endpoint.columnName === "id"
-    ))!;
-    const refreshedTarget = refreshedContext.endpoints.find((endpoint) => (
-      endpoint.tableName === "patients" && endpoint.columnName === "id"
-    ))!;
-    await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
-      sourceColumnId: refreshedSource.columnId,
-      targetColumnId: refreshedTarget.columnId,
-    }])).resolves.toBe(1);
     expect(await repository.deleteDatabaseMetadata([
       database.id,
       "99999999-9999-4999-8999-999999999999",
     ], "tables")).toBeUndefined();
     expect(await repository.listTables(database.id)).toHaveLength(2);
     expect(await repository.deleteDatabaseMetadata([database.id], "tables"))
-      .toEqual({ tables: 2, columns: 4, relationships: 2 });
+      .toEqual({ tables: 2, columns: 4, relationships: 1 });
     expect(await repository.get(database.id)).toBeDefined();
     expect(await repository.listTables(database.id)).toEqual([]);
     expect(await repository.listRelationships(database.id)).toEqual([]);
@@ -428,7 +276,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
   }
 }, 60_000);
 
-test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected and database-wide descriptions", async () => {
+test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
   const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
   const db = new Kysely<CatalogDatabase>({
     dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
@@ -522,22 +370,6 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel
       description: "Still curated visits",
       generatedDescription: "Generated visits",
     });
-
-    expect(await repository.consolidateGeneratedDescriptions(
-      database.id, "database", [],
-    )).toEqual({ copied: 3, skipped: 1 });
-    expect(await repository.getTable(database.id, visits.id)).toMatchObject({
-      description: "Generated visits",
-      generatedDescription: "Generated visits",
-    });
-    expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
-      description: "Generated id",
-      generatedDescription: "Generated id",
-    });
-    expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
-      description: "Keep patient reference",
-      generatedDescription: null,
-    });
   } finally {
     await db.destroy();
     await container.stop();
@@ -559,10 +391,6 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     await upDescriptionGeneration(db);
     await upSensitiveSuggestionRuns(db);
     await upAiTokenUsage(db);
-    await upCanonicalModelIds(db);
-    await upLocalSensitivityAnalysis(db);
-    await upSensitivityReason(db);
-    await upCatalogPreprocessingState(db);
     const repository = new KyselyCatalogRepository(db);
     const firstDatabase = await repository.create({
       workspaceId: "generation-one",
@@ -600,14 +428,14 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     const run = await repository.createDescriptionGenerationRun(
       firstDatabase.id,
       "selected_columns",
-      "openai/gpt-4.1-mini",
+      "openai-mini",
       "it",
       1,
     );
     expect(run).toMatchObject({
       databaseId: firstDatabase.id,
       scope: "selected_columns",
-      modelId: "openai/gpt-4.1-mini",
+      modelId: "openai-mini",
       language: "it",
       status: "queued",
       total: 1,
@@ -622,7 +450,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     await expect(repository.createDescriptionGenerationRun(
       secondDatabase.id,
       "selected_columns",
-      "openai/gpt-4.1-mini",
+      "openai-mini",
       "en",
       1,
     )).rejects.toThrow("A description generation run is already active");
@@ -633,7 +461,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     await expect(repository.createDescriptionGenerationRun(
       secondDatabase.id,
       "selected_columns",
-      "openai/gpt-4.1-mini",
+      "openai-mini",
       "en",
       1,
     )).rejects.toThrow("A description generation run is already active");
@@ -677,7 +505,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     const next = await repository.createDescriptionGenerationRun(
       secondDatabase.id,
       "missing",
-      "openai/gpt-4.1-mini",
+      "openai-mini",
       "en",
       1,
     );
@@ -705,7 +533,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     const allRun = await repository.createDescriptionGenerationRun(
       firstDatabase.id,
       "all",
-      "openai/gpt-4.1-mini",
+      "openai-mini",
       "it",
       2,
     );
@@ -731,10 +559,10 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     ]);
     expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined();
 
-    const suggestionRun = await repository.createSensitivityAnalysisRun(
+    const suggestionRun = await repository.createSensitiveDataSuggestionRun(
       firstDatabase.id,
       "selected_columns",
-      { engine: "local", policyVersion: "sensitivity-v1" },
+      "openai-mini",
     );
     expect(suggestionRun).toMatchObject({
       databaseId: firstDatabase.id,
@@ -742,49 +570,43 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
       total: 0,
       suggestedSensitive: 0,
       suggestedNonSensitive: 0,
-      unknown: 0,
-      engine: "local",
-      modelId: null,
-      policyVersion: "sensitivity-v1",
       startedAt: expect.any(String),
     });
-    await repository.appendSensitivityAnalysisEvent(
+    await repository.appendSensitiveDataSuggestionEvent(
       suggestionRun.id,
       "info",
       "Sensitive-field suggestion generation started.",
     );
-    await repository.appendSensitivityAnalysisEvent(
+    await repository.appendSensitiveDataSuggestionEvent(
       suggestionRun.id,
       "info",
       "Sensitive-field suggestion generation completed for 2 columns.",
     );
-    expect(await repository.updateSensitivityAnalysisRun(suggestionRun.id, {
+    expect(await repository.updateSensitiveDataSuggestionRun(suggestionRun.id, {
       status: "completed",
       total: 2,
       suggestedSensitive: 1,
-      suggestedNonSensitive: 0,
-      unknown: 1,
+      suggestedNonSensitive: 1,
       finishedAt: new Date().toISOString(),
     })).toMatchObject({
       status: "completed",
       total: 2,
       suggestedSensitive: 1,
-      suggestedNonSensitive: 0,
-      unknown: 1,
+      suggestedNonSensitive: 1,
     });
-    expect(await repository.listSensitivityAnalysisEvents(suggestionRun.id, 1)).toEqual([
+    expect(await repository.listSensitiveDataSuggestionEvents(suggestionRun.id, 1)).toEqual([
       expect.objectContaining({ sequence: 2, level: "info" }),
     ]);
-    expect((await repository.listSensitivityAnalysisRuns(1))[0]).toMatchObject({
+    expect((await repository.listSensitiveDataSuggestionRuns(1))[0]).toMatchObject({
       id: suggestionRun.id,
     });
 
-    const interruptedSuggestionRun = await repository.createSensitivityAnalysisRun(
+    const interruptedSuggestionRun = await repository.createSensitiveDataSuggestionRun(
       secondDatabase.id,
       "all",
-      { engine: "local", policyVersion: "sensitivity-v1" },
+      "openai-mini",
     );
-    expect(await repository.interruptActiveSensitivityAnalysisRuns(
+    expect(await repository.interruptActiveSensitiveDataSuggestionRuns(
       "Sensitive-field suggestion generation was interrupted by backend restart.",
     )).toEqual([
       expect.objectContaining({
@@ -835,8 +657,6 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists logical relationsh
     const target = context.endpoints.find((item) => item.tableName === "users" && item.columnName === "id")!;
     const created = await repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, false);
     expect(created).toMatchObject({ origin: "manual", status: "active" });
-    expect(await repository.getCatalogMetrics(database.id))
-      .toMatchObject({ relationships: 1 });
     await expect(repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, true))
       .resolves.toBeUndefined();
 

backend/test/catalog-runtime-binding.test.ts

@@ -1,105 +0,0 @@
-import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import { afterEach, expect, test } from "vitest";
-
-import { resolveCatalogRuntimeBinding } from "../src/catalog/runtime-binding.js";
-import type { WorkspaceDatabase } from "../src/catalog/types.js";
-import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
-import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
-
-const roots: string[] = [];
-
-afterEach(() => {
-  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
-});
-
-test("projects a Catalog database into a runtime without database data in workspace YAML", () => {
-  const root = mkdtempSync(join(tmpdir(), "thoth-catalog-runtime-"));
-  roots.push(root);
-  const secretStore = new WorkspaceSecretStore({
-    root: join(root, "vault"),
-    runtimeRoot: join(root, "runtime"),
-    installationId: "test",
-  });
-  secretStore.putMany("sales", {
-    "catalog.dwh.password": "catalog-password",
-    "evidence.signed_urls": '["https://signed.example.test/evidence"]',
-  });
-  const workspace: WorkspaceDescriptor = {
-    workspace: {
-      schema_version: 4,
-      id: "sales",
-      name: "Sales",
-      language: "en",
-    },
-    evidence: {
-      schema_version: 1,
-      source: {
-        type: "http",
-        uris: ["https://evidence.example.test/guide.md"],
-        authentication: "signed_urls_file",
-        connect_timeout_ms: 1_000,
-        read_timeout_ms: 5_000,
-        max_bytes: 10_000,
-        max_redirects: 2,
-        allow_private_hosts: false,
-        max_cache_bytes: 20_000,
-      },
-      policy: { max_chunk_chars: 4_000, retain_published_generations: 1 },
-    },
-  };
-  const database: WorkspaceDatabase = {
-    id: "database-1",
-    workspaceId: "sales",
-    engine: "postgres",
-    databaseName: "warehouse",
-    schema: "analytics",
-    version: 3,
-    createdAt: "2026-01-01T00:00:00Z",
-    updatedAt: "2026-01-01T00:00:00Z",
-    binding: {
-      transport: "postgres_direct",
-      host: "db.internal",
-      port: 5432,
-      username: "reader",
-    },
-    connectionStatus: "reachable",
-    metadataContentRevision: 7,
-    preprocessingStatus: "failed",
-  };
-
-  const lease = resolveCatalogRuntimeBinding({
-    workspace,
-    database,
-    environment: {},
-    secretRoots: [],
-    secretStore,
-  });
-  const passwordPath = lease.bindings.dwh.values.THT_WS_SALES_DWH_PASSWORD_FILE;
-  const evidencePath = lease.bindings.evidence.values.THT_WS_SALES_EVIDENCE_SIGNED_URLS_FILE;
-  try {
-    expect(workspace.dwh).toBeUndefined();
-    expect(lease.workspace.dwh).toMatchObject({
-      database: "warehouse",
-      schema: "analytics",
-      supported_transports: ["postgres_direct"],
-    });
-    expect(lease.bindings.dwh).toMatchObject({
-      transport: "postgres_direct",
-      missing: [],
-      values: {
-        THT_WS_SALES_DWH_HOST: "db.internal",
-        THT_WS_SALES_DWH_PORT: "5432",
-        THT_WS_SALES_DWH_USER: "reader",
-      },
-    });
-    expect(readFileSync(passwordPath, "utf8")).toBe("catalog-password");
-    expect(readFileSync(evidencePath, "utf8")).toContain("signed.example.test");
-  } finally {
-    lease.release();
-  }
-  expect(existsSync(passwordPath)).toBe(false);
-  expect(existsSync(evidencePath)).toBe(false);
-});

backend/test/catalog-schema-routes.test.ts

@@ -7,11 +7,7 @@ import { loadConfig } from "../src/config.js";
 import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
 import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
 import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
-import {
-  CatalogConnectorError,
-  type CatalogSyncRun,
-  type ObservedSchemaSnapshot,
-} from "../src/catalog/types.js";
+import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
 import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
 import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
@@ -20,8 +16,13 @@ const roots: string[] = [];
 afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
 
 const workspace: WorkspaceDescriptor = {
-  workspace: { schema_version: 4, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
+  workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
   dwh: { engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, supported_transports: ["postgres_direct"] },
+  semantic_index: {
+    vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
+    embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+  },
+  llm_policy: { allowed: ["zai/glm-5.2"] },
 };
 const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
 
@@ -168,32 +169,6 @@ test("synchronizes a full physical schema and derives primary and foreign key fl
   expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version);
 });
 
-test("attempts synchronization after a failed connection test and reports the live access failure", async () => {
-  const { app, repository, database, scan } = await setup();
-  await repository.recordTest(database.id, database.version, {
-    connectionStatus: "failed",
-    testedVersion: database.version,
-    lastTestedAt: new Date().toISOString(),
-    lastErrorCode: "connector_unavailable",
-    lastErrorMessage: "The database connector could not be reached or authenticated.",
-  });
-  scan.mockRejectedValueOnce(new CatalogConnectorError("upstream credentials must not escape"));
-
-  const started = await app.inject({
-    method: "POST",
-    url: `/catalog/databases/${database.id}/sync-runs`,
-    payload: { version: database.version, scope: "all", tableIds: [] },
-  });
-
-  expect(started.statusCode).toBe(202);
-  const failed = await waitFor(repository, started.json().id, "failed");
-  expect(scan).toHaveBeenCalledOnce();
-  expect(failed).toMatchObject({
-    errorCode: "schema_introspection_failed",
-    errorMessage: "The database schema could not be read. Check the connection and credentials, then try again.",
-  });
-});
-
 test("synchronizes columns for every catalog table when no table selection is supplied", async () => {
   const { app, repository, database, setObserved } = await setup();
   const tablesRun = await app.inject({
@@ -277,18 +252,13 @@ test("keeps generated descriptions editable and preserves them across synchroniz
   });
   const sensitiveOnly = await app.inject({
     method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
-    payload: {
-      version: editedColumn.json().version,
-      sensitive: true,
-      sensitivityReason: "Local assessment matched content rule pii.email.",
-    },
+    payload: { version: editedColumn.json().version, sensitive: true },
   });
   expect(sensitiveOnly.statusCode).toBe(200);
   expect(sensitiveOnly.json()).toMatchObject({
     description: "Reviewed key",
     generatedDescription: "Generated key draft",
     sensitive: true,
-    sensitivityReason: "Local assessment matched content rule pii.email.",
   });
   const emptyPatch = await app.inject({
     method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
@@ -303,7 +273,6 @@ test("keeps generated descriptions editable and preserves them across synchroniz
     description: "Reviewed key",
     generatedDescription: "Generated key draft",
     sensitive: true,
-    sensitivityReason: "Local assessment matched content rule pii.email.",
   });
 });
 
@@ -394,79 +363,6 @@ test("consolidates non-empty generated column descriptions and preserves curated
   expect(scan).not.toHaveBeenCalled();
 });
 
-test("consolidates generated descriptions for every table and column in a database", async () => {
-  const { app, repository, database, scan } = await setup();
-  await seedCatalog(repository, database);
-  const tables = await repository.listTables(database.id);
-  const patients = tables.find((table) => table.name === "patients")!;
-  const visits = tables.find((table) => table.name === "visits")!;
-  await repository.updateTableMetadata(
-    database.id,
-    patients.id,
-    patients.version,
-    "Curated patients",
-    "Generated patients",
-  );
-  const patientId = (await repository.listColumns(database.id, patients.id))[0]!;
-  const visitColumns = await repository.listColumns(database.id, visits.id);
-  const visitId = visitColumns.find((column) => column.name === "id")!;
-  const visitPatientId = visitColumns.find((column) => column.name === "patient_id")!;
-  await repository.updateColumnMetadata(
-    database.id,
-    patients.id,
-    patientId.id,
-    patientId.version,
-    "Curated patient identifier",
-    "Generated patient identifier",
-  );
-  await repository.updateColumnMetadata(
-    database.id,
-    visits.id,
-    visitId.id,
-    visitId.version,
-    "Curated visit identifier",
-    "Generated visit identifier",
-  );
-  await repository.updateColumnMetadata(
-    database.id,
-    visits.id,
-    visitPatientId.id,
-    visitPatientId.version,
-    "Keep curated patient reference",
-    "",
-  );
-
-  const response = await app.inject({
-    method: "POST",
-    url: `/catalog/databases/${database.id}/descriptions/consolidate`,
-    payload: { target: "database" },
-  });
-
-  expect(response.statusCode).toBe(200);
-  expect(response.json()).toEqual({ copied: 3, skipped: 2 });
-  expect(await repository.getTable(database.id, patients.id)).toMatchObject({
-    description: "Generated patients",
-    generatedDescription: "Generated patients",
-    version: patients.version + 2,
-  });
-  expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({
-    description: "Generated patient identifier",
-    generatedDescription: "Generated patient identifier",
-    version: patientId.version + 2,
-  });
-  expect(await repository.getColumn(database.id, visits.id, visitId.id)).toMatchObject({
-    description: "Generated visit identifier",
-    generatedDescription: "Generated visit identifier",
-    version: visitId.version + 2,
-  });
-  expect(await repository.getColumn(database.id, visits.id, visitPatientId.id)).toMatchObject({
-    description: "Keep curated patient reference",
-    generatedDescription: "",
-    version: visitPatientId.version + 1,
-  });
-  expect(scan).not.toHaveBeenCalled();
-});
-
 test("rejects description consolidation while the Workspace Database is reserved", async () => {
   const { app, repository, database, operations } = await setup();
   await seedCatalog(repository, database);
@@ -541,19 +437,9 @@ test("strictly validates description consolidation database and target ids", asy
       url: `/catalog/databases/${database.id}/descriptions/consolidate`,
       payload: { target: "tables", targetIds: [table.id], unexpected: true },
     }),
-    app.inject({
-      method: "POST",
-      url: `/catalog/databases/${database.id}/descriptions/consolidate`,
-      payload: { target: "database", targetIds: [table.id] },
-    }),
-    app.inject({
-      method: "POST",
-      url: `/catalog/databases/${database.id}/descriptions/consolidate`,
-      payload: { target: "database_columns", targetIds: [table.id] },
-    }),
   ]);
 
-  expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400, 400]);
+  expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400]);
   for (const response of responses) {
     expect(response.json()).toEqual({
       code: "description_consolidation_invalid",
@@ -643,18 +529,6 @@ test("deletes relationships for selected databases without deleting their tables
   const { app, repository, database } = await setup();
   await seedCatalog(repository, database);
   const tables = await repository.listTables(database.id);
-  const context = (await repository.getLogicalRelationshipContext(database.id))!;
-  const source = context.endpoints.find((endpoint) => (
-    endpoint.tableName === "visits" && endpoint.columnName === "id"
-  ))!;
-  const target = context.endpoints.find((endpoint) => (
-    endpoint.tableName === "patients" && endpoint.columnName === "id"
-  ))!;
-  await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
-    sourceColumnId: source.columnId,
-    targetColumnId: target.columnId,
-  }])).resolves.toBe(1);
-  await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 2 });
 
   const response = await app.inject({
     method: "POST",
@@ -663,12 +537,10 @@ test("deletes relationships for selected databases without deleting their tables
   });
 
   expect(response.statusCode).toBe(200);
-  expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 2 });
+  expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 1 });
   expect(await repository.listTables(database.id)).toHaveLength(2);
   expect(await repository.listColumns(database.id, tables[0]!.id)).not.toEqual([]);
   expect(await repository.listRelationships(database.id)).toEqual([]);
-  expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
-  await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 0 });
   expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
 });
 
@@ -804,11 +676,6 @@ test("rebuilds generated relationships and returns the exact summary", async ()
   });
   expect(first.statusCode).toBe(200);
   expect(first.json()).toEqual({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 });
-  const metrics = await app.inject({
-    method: "GET", url: `/catalog/metrics?databaseId=${database.id}`,
-  });
-  expect(metrics.statusCode).toBe(200);
-  expect(metrics.json()).toMatchObject({ relationships: 1 });
   const generated = (await repository.listLogicalRelationships(database.id))[0]!;
 
   await app.inject({

backend/test/catalog-sensitivity-analysis.test.ts

@@ -1,265 +0,0 @@
-import { expect, test, vi } from "vitest";
-import {
-  SensitivityAnalysisInterruptedError,
-  SensitivityAnalysisService,
-} from "../src/catalog/sensitivity-analysis-service.js";
-import { SensitivityAnalysisRunner } from "../src/catalog/sensitivity-analysis-runner.js";
-import type { SensitivityClassifier } from "../src/catalog/sensitivity-classifier.js";
-import type {
-  CatalogColumn,
-  CatalogRepository,
-  CatalogTable,
-  SensitivityAnalysisRun,
-  WorkspaceDatabase,
-} from "../src/catalog/types.js";
-
-const database = {
-  id: "11111111-1111-4111-8111-111111111111",
-  workspaceId: "psd-clinical",
-  engine: "postgres",
-  databaseName: "warehouse",
-  schema: "public",
-  version: 1,
-  createdAt: "2026-09-02T08:00:00Z",
-  updatedAt: "2026-09-02T08:00:00Z",
-  connectionStatus: "reachable",
-  binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
-} satisfies WorkspaceDatabase;
-
-function catalogTable(id: string, name: string): CatalogTable {
-  return {
-    id,
-    databaseId: database.id,
-    name,
-    sourceComment: null,
-    description: null,
-    generatedDescription: null,
-    lastSyncedDatabaseVersion: 1,
-    lastSyncedAt: "2026-09-02T08:00:00Z",
-    version: 1,
-    createdAt: "2026-09-02T08:00:00Z",
-    updatedAt: "2026-09-02T08:00:00Z",
-  };
-}
-
-function catalogColumn(id: string, tableId: string, name: string): CatalogColumn {
-  return {
-    id,
-    tableId,
-    name,
-    ordinalPosition: 1,
-    dataType: "text",
-    isNullable: true,
-    defaultExpression: null,
-    primaryKeyPosition: null,
-    isPrimaryKey: false,
-    isForeignKey: false,
-    foreignKeyCount: 0,
-    sourceComment: null,
-    description: null,
-    generatedDescription: null,
-    sensitive: false,
-    lastSyncedDatabaseVersion: 1,
-    lastSyncedAt: "2026-09-02T08:00:00Z",
-    version: 1,
-    createdAt: "2026-09-02T08:00:00Z",
-    updatedAt: "2026-09-02T08:00:00Z",
-  };
-}
-
-const running: SensitivityAnalysisRun = {
-  id: "22222222-2222-4222-8222-222222222222",
-  databaseId: database.id,
-  scope: "all",
-  engine: "local",
-  modelId: null,
-  policyVersion: "sensitivity-v4",
-  status: "running",
-  total: 0,
-  suggestedSensitive: 0,
-  suggestedNonSensitive: 0,
-  unknown: 0,
-  inputTokens: 0,
-  cacheReadTokens: 0,
-  outputTokens: 0,
-  createdAt: "2026-09-02T08:00:00Z",
-  startedAt: "2026-09-02T08:00:00Z",
-  updatedAt: "2026-09-02T08:00:00Z",
-  finishedAt: null,
-  errorSummary: null,
-};
-
-test("stops catalog selection when the request expires during a catalog read", async () => {
-  const controller = new AbortController();
-  const listTables = vi.fn();
-  const repository = {
-    get: vi.fn(async () => {
-      controller.abort();
-      return database;
-    }),
-    listTables,
-  } as unknown as CatalogRepository;
-  const classifier = { assess: vi.fn() } as unknown as SensitivityClassifier;
-  const analysis = new SensitivityAnalysisService(repository, classifier);
-
-  await expect(analysis.analyze(
-    database.id,
-    "all",
-    [],
-    controller.signal,
-  )).rejects.toBeInstanceOf(SensitivityAnalysisInterruptedError);
-  expect(listTables).not.toHaveBeenCalled();
-  expect(classifier.assess).not.toHaveBeenCalled();
-});
-
-test("classifies all selected tables in one breadth-first run and reports coverage", async () => {
-  const firstTable = catalogTable("33333333-3333-4333-8333-333333333333", "patients");
-  const secondTable = catalogTable("44444444-4444-4444-8444-444444444444", "encounters");
-  const firstColumn = catalogColumn(
-    "55555555-5555-4555-8555-555555555555",
-    firstTable.id,
-    "status",
-  );
-  const secondColumn = catalogColumn(
-    "66666666-6666-4666-8666-666666666666",
-    secondTable.id,
-    "note",
-  );
-  const repository = {
-    get: vi.fn(async () => database),
-    listTables: vi.fn(async () => [firstTable, secondTable]),
-    listColumns: vi.fn(async (_databaseId: string, tableId: string) => (
-      tableId === firstTable.id ? [firstColumn] : [secondColumn]
-    )),
-  } as unknown as CatalogRepository;
-  const assess = vi.fn(async (
-    _targets,
-    _signal,
-    _nerBudget,
-    onActivity?: (message: string) => void | Promise<void>,
-  ) => {
-    await onActivity?.("Scanning source data: pass 1 of 3, table batch 1 of 1.");
-    return [
-      {
-        columnId: firstColumn.id,
-        assessment: "non_sensitive" as const,
-        proposedSensitive: false,
-        evidence: [{ kind: "coverage" as const, ruleId: "coverage.sampled_1000" }],
-        observedValues: 1_000,
-        coverage: "sampled" as const,
-      },
-      {
-        columnId: secondColumn.id,
-        assessment: "sensitive" as const,
-        proposedSensitive: true,
-        evidence: [{ kind: "content" as const, ruleId: "pii.email" }],
-        observedValues: 12,
-        coverage: "sampled" as const,
-      },
-    ];
-  });
-  const classifier = { assess } as unknown as SensitivityClassifier;
-  const onPrepared = vi.fn();
-  const onProgress = vi.fn();
-  const onActivity = vi.fn();
-
-  const suggestions = await new SensitivityAnalysisService(repository, classifier).analyze(
-    database.id,
-    "all",
-    [],
-    new AbortController().signal,
-    onPrepared,
-    onProgress,
-    onActivity,
-  );
-
-  expect(assess).toHaveBeenCalledOnce();
-  expect(assess.mock.calls[0]![0]).toEqual([
-    { database, table: firstTable, columns: [firstColumn] },
-    { database, table: secondTable, columns: [secondColumn] },
-  ]);
-  expect(onPrepared).toHaveBeenCalledWith(2);
-  expect(onActivity).toHaveBeenCalledWith(
-    "Scanning source data: pass 1 of 3, table batch 1 of 1.",
-  );
-  expect(onProgress.mock.calls.map(([processed]) => processed)).toEqual([1, 2]);
-  expect(suggestions).toEqual([
-    expect.objectContaining({ columnId: firstColumn.id, sensitive: false, coverage: "sampled" }),
-    expect.objectContaining({ columnId: secondColumn.id, sensitive: true, coverage: "sampled" }),
-  ]);
-});
-
-test("persists classifier activity in the running analysis event log", async () => {
-  let persisted = running;
-  const appendEvent = vi.fn(async () => undefined);
-  const repository = {
-    get: vi.fn(async () => database),
-    createSensitivityAnalysisRun: vi.fn(async () => running),
-    getSensitivityAnalysisRun: vi.fn(async () => persisted),
-    updateSensitivityAnalysisRun: vi.fn(async (
-      _runId: string,
-      changes: Partial<SensitivityAnalysisRun>,
-    ) => {
-      persisted = { ...persisted, ...changes };
-      return persisted;
-    }),
-    appendSensitivityAnalysisEvent: appendEvent,
-  } as unknown as CatalogRepository;
-  const analysis = {
-    analyze: vi.fn(async (
-      _databaseId,
-      _scope,
-      _targetIds,
-      _signal,
-      onPrepared,
-      _onProgress,
-      onActivity,
-    ) => {
-      await onPrepared?.(0);
-      await onActivity?.("Scanning source data: pass 1 of 3, table batch 1 of 1.");
-      return [];
-    }),
-  } as unknown as SensitivityAnalysisService;
-
-  await new SensitivityAnalysisRunner(repository, analysis).run(
-    database.id,
-    "all",
-    [],
-    new AbortController().signal,
-  );
-
-  expect(appendEvent).toHaveBeenCalledWith(
-    running.id,
-    "info",
-    "Scanning source data: pass 1 of 3, table batch 1 of 1.",
-  );
-});
-
-test("marks a created run interrupted if the request deadline expires during persistence", async () => {
-  const controller = new AbortController();
-  const update = vi.fn(async (_runId: string, changes: Partial<SensitivityAnalysisRun>) => ({
-    ...running,
-    ...changes,
-  }));
-  const repository = {
-    get: vi.fn(async () => database),
-    createSensitivityAnalysisRun: vi.fn(async () => {
-      controller.abort();
-      return running;
-    }),
-    updateSensitivityAnalysisRun: update,
-    appendSensitivityAnalysisEvent: vi.fn(async () => undefined),
-  } as unknown as CatalogRepository;
-  const analysis = { analyze: vi.fn() } as unknown as SensitivityAnalysisService;
-  const runner = new SensitivityAnalysisRunner(repository, analysis);
-
-  await expect(runner.run(database.id, "all", [], controller.signal))
-    .rejects.toBeInstanceOf(SensitivityAnalysisInterruptedError);
-  expect(analysis.analyze).not.toHaveBeenCalled();
-  expect(update).toHaveBeenCalledWith(running.id, expect.objectContaining({
-    status: "interrupted",
-    total: 0,
-    unknown: 0,
-    errorSummary: "Local sensitivity analysis was interrupted before completion.",
-  }));
-});

backend/test/catalog-sensitivity-classifier.test.ts

@@ -1,706 +0,0 @@
-import { expect, test, vi } from "vitest";
-import {
-  SensitivityClassifier,
-  type LocalNerDetector,
-  type SensitivityNerBudget,
-  type SensitivityTableScan,
-  type SensitivityValueSource,
-} from "../src/catalog/sensitivity-classifier.js";
-import type { CatalogColumn, CatalogTable, WorkspaceDatabase } from "../src/catalog/types.js";
-import { CatalogConnectorError } from "../src/catalog/types.js";
-
-const database = {
-  id: "11111111-1111-4111-8111-111111111111",
-  workspaceId: "psd-clinical",
-  engine: "postgres",
-  databaseName: "warehouse",
-  schema: "public",
-  version: 1,
-  createdAt: "2026-09-02T08:00:00Z",
-  updatedAt: "2026-09-02T08:00:00Z",
-  connectionStatus: "reachable",
-  binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
-} satisfies WorkspaceDatabase;
-
-const table = {
-  id: "22222222-2222-4222-8222-222222222222",
-  databaseId: database.id,
-  name: "observations",
-  sourceComment: null,
-  description: null,
-  generatedDescription: null,
-  lastSyncedDatabaseVersion: 1,
-  lastSyncedAt: "2026-09-02T08:00:00Z",
-  version: 1,
-  createdAt: "2026-09-02T08:00:00Z",
-  updatedAt: "2026-09-02T08:00:00Z",
-} satisfies CatalogTable;
-
-function column(overrides: Partial<CatalogColumn> = {}): CatalogColumn {
-  return {
-    id: "33333333-3333-4333-8333-333333333333",
-    tableId: table.id,
-    name: "note",
-    ordinalPosition: 1,
-    dataType: "character varying",
-    isNullable: true,
-    defaultExpression: null,
-    primaryKeyPosition: null,
-    isPrimaryKey: false,
-    isForeignKey: false,
-    foreignKeyCount: 0,
-    sourceComment: null,
-    description: null,
-    generatedDescription: null,
-    sensitive: false,
-    lastSyncedDatabaseVersion: 1,
-    lastSyncedAt: "2026-09-02T08:00:00Z",
-    version: 1,
-    createdAt: "2026-09-02T08:00:00Z",
-    updatedAt: "2026-09-02T08:00:00Z",
-    ...overrides,
-  };
-}
-
-function source(scan: SensitivityTableScan): SensitivityValueSource {
-  return { scanTable: vi.fn(async (_request, consume) => {
-    for (const batch of scan.batches) await consume(batch);
-    return scan.coverage;
-  }) };
-}
-
-test("reports source-scan activity before a long table scan completes", async () => {
-  let releaseScan!: () => void;
-  const scanGate = new Promise<void>((resolve) => {
-    releaseScan = resolve;
-  });
-  const scanTable = vi.fn(async () => {
-    await scanGate;
-    return { kind: "complete" as const, observedValues: 0 };
-  });
-  const activity = vi.fn();
-  const analysis = new SensitivityClassifier({ scanTable }).assess(
-    [{ database, table, columns: [column()] }],
-    new AbortController().signal,
-    undefined,
-    activity,
-  );
-
-  await vi.waitFor(() => expect(scanTable).toHaveBeenCalledOnce());
-  releaseScan();
-  await analysis;
-
-  expect(activity).toHaveBeenCalledWith(
-    "Scanning source data: pass 1 of 3, table batch 1 of 1.",
-  );
-});
-
-test("one email hidden in a generically named column makes the whole column sensitive", async () => {
-  const target = column();
-  const values = source({
-    batches: [[
-      { columnId: target.id, value: "nessun contatto", characterLength: 16 },
-      { columnId: target.id, value: "mario.rossi@example.it", characterLength: 23 },
-    ]],
-    coverage: { kind: "complete", observedValues: 2 },
-  });
-  const classifier = new SensitivityClassifier(values);
-
-  const [assessment] = await classifier.assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    columnId: target.id,
-    assessment: "sensitive",
-    proposedSensitive: true,
-    evidence: [{ kind: "content", ruleId: "pii.email" }],
-  });
-});
-
-test("one text value longer than 500 characters makes the whole column sensitive", async () => {
-  const target = column({ name: "comment" });
-  const values = source({
-    batches: [[{ columnId: target.id, value: "x".repeat(501), characterLength: 743 }]],
-    coverage: { kind: "sampled", observedValues: 1 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "sensitive",
-    proposedSensitive: true,
-    evidence: [{ kind: "length", ruleId: "text.over_500_characters" }],
-  });
-});
-
-test("scans every table at 300 before advancing to 1,000 and 3,000 values", async () => {
-  const otherTable = { ...table, id: "77777777-7777-4777-8777-777777777777", name: "events" };
-  const first = column({ name: "status" });
-  const second = column({
-    id: "88888888-8888-4888-8888-888888888888",
-    tableId: otherTable.id,
-    name: "comment",
-  });
-  const calls: string[] = [];
-  const values: SensitivityValueSource = {
-    scanTable: vi.fn(async (request, consume) => {
-      calls.push(`${request.table.name}:${request.valuesPerColumn}:${request.sampleOffset}`);
-      await consume(request.columns.map((item) => ({
-        columnId: item.id,
-        value: "ordinary",
-        characterLength: 8,
-      })));
-      return { kind: "sampled", observedValues: request.columns.length };
-    }),
-  };
-
-  await new SensitivityClassifier(values).assess([
-    { database, table, columns: [first] },
-    { database, table: otherTable, columns: [second] },
-  ], new AbortController().signal);
-
-  expect(calls).toEqual([
-    "observations:300:0",
-    "events:300:0",
-    "observations:700:300",
-    "events:700:300",
-    "observations:2000:1000",
-    "events:2000:1000",
-  ]);
-});
-
-test("runs at most two table scans concurrently", async () => {
-  const targets = Array.from({ length: 3 }, (_, index) => {
-    const targetTable = {
-      ...table,
-      id: `00000000-0000-4000-8000-${(index + 1).toString().padStart(12, "0")}`,
-      name: `table_${index + 1}`,
-    };
-    return {
-      database,
-      table: targetTable,
-      columns: [column({
-        id: `10000000-0000-4000-8000-${(index + 1).toString().padStart(12, "0")}`,
-        tableId: targetTable.id,
-        name: `attribute_${index + 1}`,
-      })],
-    };
-  });
-  let active = 0;
-  let maximum = 0;
-  const values: SensitivityValueSource = {
-    scanTable: vi.fn(async () => {
-      active += 1;
-      maximum = Math.max(maximum, active);
-      await Promise.resolve();
-      active -= 1;
-      return { kind: "complete", observedValues: 0 };
-    }),
-  };
-
-  await new SensitivityClassifier(values).assess(targets, new AbortController().signal);
-
-  expect(maximum).toBe(2);
-  expect(values.scanTable).toHaveBeenCalledTimes(3);
-});
-
-test("aborts a peer table scan when another concurrent source scan fails", async () => {
-  const otherTable = { ...table, id: "77777777-7777-4777-8777-777777777777", name: "events" };
-  const first = column({ name: "status" });
-  const second = column({
-    id: "88888888-8888-4888-8888-888888888888",
-    tableId: otherTable.id,
-    name: "comment",
-  });
-  let peerSignal: AbortSignal | undefined;
-  const failure = new CatalogConnectorError("source unavailable");
-  const values: SensitivityValueSource = {
-    scanTable: vi.fn(async (request, _consume, scanSignal) => {
-      if (request.table.id === table.id) {
-        await Promise.resolve();
-        throw failure;
-      }
-      peerSignal = scanSignal;
-      return await new Promise((_resolve, reject) => {
-        scanSignal.addEventListener("abort", () => reject(scanSignal.reason), { once: true });
-      });
-    }),
-  };
-
-  await expect(new SensitivityClassifier(values).assess([
-    { database, table, columns: [first] },
-    { database, table: otherTable, columns: [second] },
-  ], new AbortController().signal)).rejects.toBe(failure);
-
-  expect(peerSignal?.aborted).toBe(true);
-});
-
-test("stops sampling a column as soon as one value is sensitive", async () => {
-  const target = column();
-  const values: SensitivityValueSource = {
-    scanTable: vi.fn(async (request, consume) => {
-      await consume([{ columnId: target.id, value: "mario.rossi@example.it", characterLength: 23 }]);
-      return { kind: "sampled", observedValues: 1 };
-    }),
-  };
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(values.scanTable).toHaveBeenCalledOnce();
-  expect(assessment).toMatchObject({ assessment: "sensitive", proposedSensitive: true });
-});
-
-test("stops non-text columns after the 1,000-value stage", async () => {
-  const target = column({ dataType: "integer", name: "sequence_number" });
-  const values: SensitivityValueSource = {
-    scanTable: vi.fn(async (request, consume) => {
-      await consume([{ columnId: target.id, value: "42", characterLength: 2 }]);
-      return { kind: "sampled", observedValues: 1 };
-    }),
-  };
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(vi.mocked(values.scanTable).mock.calls.map(([request]) => request.valuesPerColumn))
-    .toEqual([300, 700]);
-  expect(assessment).toMatchObject({
-    assessment: "non_sensitive",
-    proposedSensitive: false,
-    coverage: "sampled",
-    evidence: [{ kind: "coverage", ruleId: "coverage.sampled_1000" }],
-  });
-});
-
-test("complete coverage classifies benign and empty columns as non-sensitive", async () => {
-  const benign = column({ id: "44444444-4444-4444-8444-444444444444", name: "status" });
-  const empty = column({ id: "55555555-5555-4555-8555-555555555555", name: "optional_note" });
-  const humanProtected = column({
-    id: "66666666-6666-4666-8666-666666666666",
-    name: "category",
-    sensitive: true,
-  });
-  const values = source({
-    batches: [[
-      { columnId: benign.id, value: "active", characterLength: 6 },
-      { columnId: empty.id, value: null, characterLength: null },
-      { columnId: humanProtected.id, value: "administrative", characterLength: 14 },
-    ]],
-    coverage: { kind: "complete", observedValues: 1 },
-  });
-
-  const assessments = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [benign, empty, humanProtected] },
-    new AbortController().signal,
-  );
-
-  expect(assessments).toEqual([
-    expect.objectContaining({ columnId: benign.id, assessment: "non_sensitive", proposedSensitive: false }),
-    expect.objectContaining({
-      columnId: empty.id,
-      assessment: "non_sensitive",
-      proposedSensitive: false,
-      evidence: [{ kind: "coverage", ruleId: "coverage.no_values" }],
-    }),
-    expect.objectContaining({
-      columnId: humanProtected.id,
-      assessment: "non_sensitive",
-      proposedSensitive: false,
-    }),
-  ]);
-});
-
-test("sampled coverage without a match proposes non-sensitive independently of the current flag", async () => {
-  const target = column({ sensitive: true });
-  const values = source({
-    batches: [[{ columnId: target.id, value: "ordinary", characterLength: 8 }]],
-    coverage: { kind: "sampled", observedValues: 1 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "non_sensitive",
-    proposedSensitive: false,
-    evidence: [{ kind: "coverage", ruleId: "coverage.sampled_3000" }],
-  });
-});
-
-test("an unavailable source fails the analysis instead of producing unknown decisions", async () => {
-  const unresolved = column();
-  const metadataMatch = column({
-    id: "44444444-4444-4444-8444-444444444444",
-    name: "codice_fiscale",
-  });
-  const values: SensitivityValueSource = {
-    scanTable: vi.fn(async () => {
-      throw new CatalogConnectorError("upstream detail must not escape");
-    }),
-  };
-
-  await expect(new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [unresolved, metadataMatch] },
-    new AbortController().signal,
-  )).rejects.toBeInstanceOf(CatalogConnectorError);
-});
-
-test("strong Italian PII metadata is sensitive even when the source column is empty", async () => {
-  const target = column({ name: "codice_fiscale" });
-  const values = source({
-    batches: [],
-    coverage: { kind: "complete", observedValues: 0 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "sensitive",
-    proposedSensitive: true,
-    evidence: [{ kind: "metadata", ruleId: "metadata.direct_identifier" }],
-  });
-  expect(values.scanTable).not.toHaveBeenCalled();
-});
-
-test("excludes bigint primary keys from content analysis as non-informative identifiers", async () => {
-  const target = column({
-    name: "id",
-    dataType: "bigint",
-    primaryKeyPosition: 1,
-    isPrimaryKey: true,
-  });
-  const values = source({
-    batches: [[{
-      columnId: target.id,
-      value: "3471234567",
-      characterLength: 10,
-    }]],
-    coverage: { kind: "complete", observedValues: 1 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "non_sensitive",
-    proposedSensitive: false,
-    evidence: [{
-      kind: "type",
-      ruleId: "type.bigint_primary_key_non_informative",
-      label: "non-informative bigint primary key",
-    }],
-    coverage: "metadata",
-  });
-  expect(values.scanTable).not.toHaveBeenCalled();
-});
-
-test("infers an undeclared bigint column named pk as a non-informative primary-key identifier", async () => {
-  const target = column({
-    name: "pk",
-    dataType: "bigint",
-    primaryKeyPosition: null,
-    isPrimaryKey: false,
-  });
-  const values = source({
-    batches: [[{
-      columnId: target.id,
-      value: "3471234567",
-      characterLength: 10,
-    }]],
-    coverage: { kind: "complete", observedValues: 1 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "non_sensitive",
-    proposedSensitive: false,
-    evidence: [{
-      kind: "metadata",
-      ruleId: "metadata.bigint_pk_identifier_non_informative",
-      label: "non-informative conventional bigint primary-key identifier",
-    }],
-    coverage: "metadata",
-  });
-  expect(values.scanTable).not.toHaveBeenCalled();
-});
-
-test("still inspects phone-like values in bigint columns that are not primary keys", async () => {
-  const target = column({ name: "id", dataType: "bigint" });
-  const values = source({
-    batches: [[{
-      columnId: target.id,
-      value: "3471234567",
-      characterLength: 10,
-    }]],
-    coverage: { kind: "complete", observedValues: 1 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "sensitive",
-    proposedSensitive: true,
-    evidence: [{ kind: "content", ruleId: "pii.phone_number" }],
-  });
-  expect(values.scanTable).toHaveBeenCalledOnce();
-});
-
-test.each([
-  ["RSSMRA85T10A562S", "pii.italian_fiscal_code"],
-  ["IT60 X054 2811 1010 0000 0123 456", "financial.iban"],
-  ["4111 1111 1111 1111", "financial.payment_card"],
-  ["SWIFT DEUTDEFF500", "financial.bic"],
-  ["Partita IVA 00743110157", "pii.italian_vat"],
-  ["Passaporto YA1234567", "pii.passport_number"],
-  ["Carta d'identità CA12345AA", "pii.identity_card"],
-  ["Patente di guida U11234567A", "pii.drivers_license_number"],
-  ["Chiamare +39 347 123 4567", "pii.phone_number"],
-  ["Client 192.168.1.5", "network.ip_address"],
-  ["Device 00:1B:44:11:3A:B7", "network.mac_address"],
-  ["https://example.org/profiles/mario", "network.url"],
-  ["550e8400-e29b-41d4-a716-446655440000", "pii.uuid"],
-  ["AWS key AKIAIOSFODNN7EXAMPLE", "credential.access_key"],
-  ["Diagnosi: carcinoma mammario con metastasi ossee", "health.clinical_term"],
-  ["-----BEGIN PRIVATE KEY----- secret -----END PRIVATE KEY-----", "credential.private_key"],
-  ['{"profile":{"email":"not yet supplied"}}', "pii.json_sensitive_key"],
-] as const)("recognizes validated sensitive content without relying on the column name: %s", async (
-  value,
-  ruleId,
-) => {
-  const target = column();
-  const values = source({
-    batches: [[{ columnId: target.id, value, characterLength: value.length }]],
-    coverage: { kind: "complete", observedValues: 1 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "sensitive",
-    evidence: [{ kind: "content", ruleId }],
-  });
-});
-
-test("does not make a malformed email decisive", async () => {
-  const target = column();
-  const [assessment] = await new SensitivityClassifier(source({
-    batches: [[{
-      columnId: target.id,
-      value: "contatto a@b..com non valido",
-      characterLength: 28,
-    }]],
-    coverage: { kind: "complete", observedValues: 1 },
-  })).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "non_sensitive",
-    evidence: [{ kind: "coverage", ruleId: "coverage.complete" }],
-  });
-});
-
-test("finds a valid email after a malformed candidate in the same value", async () => {
-  const target = column();
-  const [assessment] = await new SensitivityClassifier(source({
-    batches: [[{
-      columnId: target.id,
-      value: "contatto a@b..com; indirizzo valido mario.rossi@example.it",
-      characterLength: 58,
-    }]],
-    coverage: { kind: "complete", observedValues: 1 },
-  })).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "sensitive",
-    evidence: [{ kind: "content", ruleId: "pii.email" }],
-  });
-});
-
-test("optional local NER evidence can make otherwise ambiguous Italian text sensitive", async () => {
-  const target = column();
-  const values = source({
-    batches: [[{ columnId: target.id, value: "Dimesso Mario Rossi", characterLength: 19 }]],
-    coverage: { kind: "sampled", observedValues: 1 },
-  });
-  const detector: LocalNerDetector = {
-    detect: vi.fn(async () => [{ columnId: target.id, label: "person_name", confidence: 0.91 }]),
-  };
-
-  const [assessment] = await new SensitivityClassifier(values, detector).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(detector.detect).toHaveBeenCalledWith(
-    [{ columnId: target.id, text: "Dimesso Mario Rossi" }],
-    expect.any(AbortSignal),
-    expect.any(Number),
-  );
-  expect(assessment).toMatchObject({
-    assessment: "sensitive",
-    evidence: [{ kind: "ner", ruleId: "ner.entity", label: "person_name", confidence: 0.91 }],
-  });
-});
-
-test("does not wait for an optional NER worker that is still warming", async () => {
-  const target = column();
-  const detector: LocalNerDetector = {
-    isReady: () => false,
-    detect: vi.fn(async () => [{ columnId: target.id, label: "person", confidence: 0.99 }]),
-  };
-
-  const [assessment] = await new SensitivityClassifier(source({
-    batches: [[{ columnId: target.id, value: "Dimesso Mario Rossi", characterLength: 19 }]],
-    coverage: { kind: "sampled", observedValues: 1 },
-  }), detector).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(detector.detect).not.toHaveBeenCalled();
-  expect(assessment).toMatchObject({
-    assessment: "non_sensitive",
-    evidence: [{ kind: "coverage", ruleId: "coverage.sampled_3000" }],
-  });
-});
-
-test("bounds each optional NER request when an installation raises the per-table work limit", async () => {
-  const columns = Array.from({ length: 17 }, (_, index) => column({
-    id: `00000000-0000-4000-8000-${(index + 1).toString(16).padStart(12, "0")}`,
-    name: `attribute_${index + 1}`,
-    ordinalPosition: index + 1,
-  }));
-  const observations = columns.flatMap((item, columnIndex) => Array.from(
-    { length: 8 },
-    (_, valueIndex) => ({
-      columnId: item.id,
-      value: `ordinary-${columnIndex}-${valueIndex}`,
-      characterLength: 13,
-    }),
-  ));
-  const detector: LocalNerDetector = { detect: vi.fn(async () => []) };
-
-  await new SensitivityClassifier(source({
-    batches: [observations],
-    coverage: { kind: "complete", observedValues: 8 },
-  }), detector, { maxNerCandidatesPerTable: 136 }).assessTable(
-    { database, table, columns },
-    new AbortController().signal,
-  );
-
-  expect(detector.detect).toHaveBeenCalledTimes(2);
-  expect(vi.mocked(detector.detect).mock.calls.map(([candidates]) => candidates.length)).toEqual([
-    128,
-    8,
-  ]);
-});
-
-test("limits default NER work to two candidates spread across a wide table", async () => {
-  const columns = Array.from({ length: 10 }, (_, index) => column({
-    id: `10000000-0000-4000-8000-${(index + 1).toString(16).padStart(12, "0")}`,
-    name: `attribute_${index + 1}`,
-    ordinalPosition: index + 1,
-  }));
-  const detector: LocalNerDetector = { detect: vi.fn(async () => []) };
-
-  await new SensitivityClassifier(source({
-    batches: [columns.flatMap((item, columnIndex) => [0, 1].map((valueIndex) => ({
-      columnId: item.id,
-      value: `ordinary-${columnIndex}-${valueIndex}`,
-      characterLength: 13,
-    })))],
-    coverage: { kind: "complete", observedValues: 2 },
-  }), detector).assessTable(
-    { database, table, columns },
-    new AbortController().signal,
-  );
-
-  expect(detector.detect).toHaveBeenCalledOnce();
-  const submitted = vi.mocked(detector.detect).mock.calls[0]![0];
-  expect(submitted).toHaveLength(2);
-  expect(new Set(submitted.map((candidate) => candidate.columnId)).size).toBe(2);
-});
-
-test("shares a bounded NER time allowance across tables in one analysis run", async () => {
-  const target = column();
-  const values = source({
-    batches: [[{ columnId: target.id, value: "Dimesso Mario Rossi", characterLength: 19 }]],
-    coverage: { kind: "sampled", observedValues: 1 },
-  });
-  const detector: LocalNerDetector = {
-    detect: vi.fn(async () => {
-      await new Promise((resolve) => setTimeout(resolve, 20));
-      return [];
-    }),
-  };
-  const classifier = new SensitivityClassifier(values, detector);
-  const nerBudget: SensitivityNerBudget = { remainingMs: 1 };
-
-  await classifier.assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-    Date.now() + 1_000,
-    nerBudget,
-  );
-  await classifier.assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-    Date.now() + 1_000,
-    nerBudget,
-  );
-
-  expect(detector.detect).toHaveBeenCalledOnce();
-  expect(nerBudget.remainingMs).toBe(0);
-});
-
-test("uninterpretable binary content is protected conservatively without scanning", async () => {
-  const target = column({ dataType: "bytea" });
-  const values = source({
-    batches: [[{ columnId: target.id, value: "\\xdeadbeef", characterLength: 10 }]],
-    coverage: { kind: "complete", observedValues: 1 },
-  });
-
-  const [assessment] = await new SensitivityClassifier(values).assessTable(
-    { database, table, columns: [target] },
-    new AbortController().signal,
-  );
-
-  expect(assessment).toMatchObject({
-    assessment: "sensitive",
-    proposedSensitive: true,
-    evidence: [{ kind: "type", ruleId: "type.binary_uninspectable" }],
-  });
-  expect(values.scanTable).not.toHaveBeenCalled();
-});

backend/test/catalog-sensitivity-value-source.test.ts

@@ -1,305 +0,0 @@
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { expect, test, vi } from "vitest";
-import type { CatalogDatabaseClient, CatalogPostgresAccess } from "../src/catalog/postgres-access.js";
-import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
-import { ConcreteSensitivityValueSource } from "../src/catalog/sensitivity-value-source.js";
-import {
-  CatalogConnectorError,
-  type CatalogColumn,
-  type CatalogTable,
-  type WorkspaceDatabase,
-} from "../src/catalog/types.js";
-import type { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
-
-const database = {
-  id: "11111111-1111-4111-8111-111111111111",
-  workspaceId: "psd-clinical",
-  engine: "postgres",
-  databaseName: "warehouse",
-  schema: 'clinical"data',
-  version: 1,
-  createdAt: "2026-09-02T08:00:00Z",
-  updatedAt: "2026-09-02T08:00:00Z",
-  connectionStatus: "reachable",
-  binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
-} satisfies WorkspaceDatabase;
-
-const table = {
-  id: "22222222-2222-4222-8222-222222222222",
-  databaseId: database.id,
-  name: 'patient"facts',
-  sourceComment: null,
-  description: null,
-  generatedDescription: null,
-  lastSyncedDatabaseVersion: 1,
-  lastSyncedAt: "2026-09-02T08:00:00Z",
-  version: 1,
-  createdAt: "2026-09-02T08:00:00Z",
-  updatedAt: "2026-09-02T08:00:00Z",
-} satisfies CatalogTable;
-
-function column(id: string, name: string): CatalogColumn {
-  return {
-    id,
-    tableId: table.id,
-    name,
-    ordinalPosition: 1,
-    dataType: "text",
-    isNullable: true,
-    defaultExpression: null,
-    primaryKeyPosition: null,
-    isPrimaryKey: false,
-    isForeignKey: false,
-    foreignKeyCount: 0,
-    sourceComment: null,
-    description: null,
-    generatedDescription: null,
-    sensitive: false,
-    lastSyncedDatabaseVersion: 1,
-    lastSyncedAt: "2026-09-02T08:00:00Z",
-    version: 1,
-    createdAt: "2026-09-02T08:00:00Z",
-    updatedAt: "2026-09-02T08:00:00Z",
-  };
-}
-
-function request(columns: readonly CatalogColumn[], overrides: Record<string, unknown> = {}) {
-  return {
-    database,
-    table,
-    columns,
-    valuesPerColumn: 300,
-    sampleOffset: 0,
-    sampleSeed: 37,
-    queryTimeoutMs: 5_000,
-    fullScanThreshold: 1_000,
-    ...overrides,
-  };
-}
-
-test("uses bounded read-only PostgreSQL sampling for tables above 1,000 rows", async () => {
-  const note = column("33333333-3333-4333-8333-333333333333", "note");
-  const contact = column("44444444-4444-4444-8444-444444444444", 'contact"value');
-  const query = vi.fn(async (sql: string) => {
-    if (sql.startsWith("SELECT 1 AS __present")) {
-      return { rows: Array.from({ length: 1_001 }, () => ({ __present: 1 })) };
-    }
-    if (sql.startsWith("WITH sampled")) {
-      return { rows: [
-        { __column_index: 0, __value: "ordinary", __length: "8" },
-        { __column_index: 1, __value: "mario.rossi@example.it", __length: 23 },
-      ] };
-    }
-    return { rows: [] };
-  });
-  const end = vi.fn(async () => undefined);
-  const access: CatalogPostgresAccess = {
-    connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
-  };
-  const consume = vi.fn();
-
-  await expect(new ConcreteSensitivityValueSource(access).scanTable(
-    request([note, contact]),
-    consume,
-    new AbortController().signal,
-  )).resolves.toEqual({ kind: "sampled", observedValues: 2 });
-
-  expect(query.mock.calls[0]).toEqual(["BEGIN TRANSACTION READ ONLY", []]);
-  expect(query).toHaveBeenCalledWith("SELECT set_config('statement_timeout', $1, true)", ["5000ms"]);
-  const sampleSql = query.mock.calls.map(([sql]) => String(sql)).find((sql) => sql.startsWith("WITH sampled"));
-  expect(sampleSql).toContain('FROM "clinical""data"."patient""facts" TABLESAMPLE SYSTEM (30)');
-  expect(sampleSql).toContain("REPEATABLE (37)");
-  expect(sampleSql).toContain("LIMIT 3000 OFFSET 0");
-  expect(sampleSql).toContain("CROSS JOIN LATERAL");
-  expect(sampleSql).toContain("WHERE __rank <= 300");
-  expect(consume).toHaveBeenCalledWith([
-    { columnId: note.id, value: "ordinary", characterLength: 8 },
-    { columnId: contact.id, value: "mario.rossi@example.it", characterLength: 23 },
-  ]);
-  expect(query.mock.calls.at(-1)).toEqual(["ROLLBACK", []]);
-  expect(end).toHaveBeenCalledOnce();
-});
-
-test("fully scans a table when the 1,001-row probe proves it is small", async () => {
-  const note = column("33333333-3333-4333-8333-333333333333", "note");
-  const query = vi.fn(async (sql: string) => {
-    if (sql.startsWith("SELECT 1 AS __present")) return { rows: [{ __present: 1 }] };
-    if (sql.startsWith("WITH sampled")) {
-      return { rows: [{ __column_index: 0, __value: "ordinary", __length: 8 }] };
-    }
-    return { rows: [] };
-  });
-  const access: CatalogPostgresAccess = {
-    connect: vi.fn(async () => ({ query, end: vi.fn(async () => undefined) }) as CatalogDatabaseClient),
-  };
-  const consume = vi.fn();
-
-  await expect(new ConcreteSensitivityValueSource(access).scanTable(
-    request([note]),
-    consume,
-    new AbortController().signal,
-  )).resolves.toEqual({ kind: "complete", observedValues: 1 });
-
-  const valueSql = query.mock.calls.map(([sql]) => String(sql)).find((sql) => sql.startsWith("WITH sampled"));
-  expect(valueSql).not.toContain("TABLESAMPLE");
-  expect(valueSql).toContain("WHERE __rank <= 1000");
-  expect(consume).toHaveBeenCalledWith([
-    { columnId: note.id, value: "ordinary", characterLength: 8 },
-  ]);
-});
-
-test("falls back to sampling when the small-table probe reaches its query timeout", async () => {
-  const note = column("33333333-3333-4333-8333-333333333333", "note");
-  const query = vi.fn(async (sql: string) => {
-    if (sql.startsWith("SELECT 1 AS __present")) {
-      throw Object.assign(new Error("statement timeout"), { code: "57014" });
-    }
-    if (sql.startsWith("WITH sampled")) {
-      return { rows: [{ __column_index: 0, __value: "sample", __length: 6 }] };
-    }
-    return { rows: [] };
-  });
-  const access: CatalogPostgresAccess = {
-    connect: vi.fn(async () => ({ query, end: vi.fn(async () => undefined) }) as CatalogDatabaseClient),
-  };
-  const consume = vi.fn();
-
-  await expect(new ConcreteSensitivityValueSource(access).scanTable(
-    request([note]),
-    consume,
-    new AbortController().signal,
-  )).resolves.toEqual({ kind: "sampled", observedValues: 1 });
-  expect(query.mock.calls.map(([sql]) => String(sql))).toContain(
-    "ROLLBACK TO SAVEPOINT sensitivity_scan_1",
-  );
-});
-
-test("limits each source query to at most 25 columns", async () => {
-  const columns = Array.from({ length: 26 }, (_, index) => column(
-    `00000000-0000-4000-8000-${(index + 1).toString().padStart(12, "0")}`,
-    `attribute_${index + 1}`,
-  ));
-  const query = vi.fn(async (sql: string) => {
-    if (sql.startsWith("SELECT 1 AS __present")) {
-      return { rows: Array.from({ length: 1_001 }, () => ({ __present: 1 })) };
-    }
-    if (sql.startsWith("WITH sampled")) {
-      return { rows: [{ __column_index: 0, __value: "ordinary", __length: 8 }] };
-    }
-    return { rows: [] };
-  });
-  const access: CatalogPostgresAccess = {
-    connect: vi.fn(async () => ({ query, end: vi.fn(async () => undefined) }) as CatalogDatabaseClient),
-  };
-
-  await new ConcreteSensitivityValueSource(access).scanTable(
-    request(columns),
-    vi.fn(),
-    new AbortController().signal,
-  );
-
-  expect(query.mock.calls.filter(([sql]) => String(sql).startsWith("WITH sampled"))).toHaveLength(2);
-});
-
-test("scans a REST run_query binding without PostgreSQL-wire access", async () => {
-  const root = mkdtempSync(join(tmpdir(), "tht-sensitivity-rest-"));
-  const credentialFile = join(root, "api-key");
-  writeFileSync(credentialFile, "test-api-key\n", { mode: 0o600 });
-  const release = vi.fn();
-  const secretStore = {
-    materialize: vi.fn(() => ({
-      files: new Map([[CATALOG_SECRET_IDS.apiKey, credentialFile]]),
-      release,
-    })),
-  } as unknown as WorkspaceSecretStore;
-  const fetchMock = vi.fn(async () => new Response(JSON.stringify([
-    { __column_index: 0, __value: "mario.rossi@example.it", __length: 23 },
-  ]), { status: 200, headers: { "content-type": "application/json" } }));
-  vi.stubGlobal("fetch", fetchMock);
-  const access: CatalogPostgresAccess = {
-    connect: vi.fn(async () => { throw new Error("PostgreSQL access must not be used"); }),
-  };
-  const values = new ConcreteSensitivityValueSource(access, secretStore);
-  const restDatabase: WorkspaceDatabase = {
-    ...database,
-    binding: {
-      transport: "rest_api",
-      baseUrl: "https://dwh.example.test/root/",
-      restPath: "/health",
-      restAuth: "x-api-key",
-    },
-  };
-  const note = column("33333333-3333-4333-8333-333333333333", "note");
-  const consume = vi.fn();
-
-  try {
-    await expect(values.scanTable(request([note], {
-      database: restDatabase,
-      fullScanThreshold: undefined,
-    }), consume, new AbortController().signal)).resolves.toEqual({
-      kind: "sampled",
-      observedValues: 1,
-    });
-    expect(access.connect).not.toHaveBeenCalled();
-    expect(fetchMock).toHaveBeenCalledWith(
-      "https://dwh.example.test/root/rpc/run_query",
-      expect.objectContaining({
-        method: "POST",
-        headers: { "content-type": "application/json", "x-api-key": "test-api-key" },
-      }),
-    );
-    const body = JSON.parse(String(fetchMock.mock.calls[0]![1]!.body));
-    expect(body.query_text).toContain('FROM "clinical""data"."patient""facts" TABLESAMPLE SYSTEM (30)');
-    expect(consume).toHaveBeenCalledWith([
-      { columnId: note.id, value: "mario.rossi@example.it", characterLength: 23 },
-    ]);
-    expect(release).toHaveBeenCalledOnce();
-  } finally {
-    vi.unstubAllGlobals();
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("falls back to a sequential bounded sample when randomized sampling times out", async () => {
-  const note = column("33333333-3333-4333-8333-333333333333", "note");
-  const query = vi.fn(async (sql: string) => {
-    if (sql.startsWith("WITH sampled") && sql.includes("TABLESAMPLE")) {
-      throw Object.assign(new Error("raw source detail"), { code: "57014" });
-    }
-    if (sql.startsWith("WITH sampled")) {
-      return { rows: [{ __column_index: 0, __value: "ordinary", __length: 8 }] };
-    }
-    return { rows: [] };
-  });
-  const access: CatalogPostgresAccess = {
-    connect: vi.fn(async () => ({ query, end: vi.fn(async () => undefined) }) as CatalogDatabaseClient),
-  };
-
-  await expect(new ConcreteSensitivityValueSource(access).scanTable(
-    request([note], { fullScanThreshold: undefined }),
-    vi.fn(),
-    new AbortController().signal,
-  )).resolves.toEqual({ kind: "sampled", observedValues: 1 });
-  expect(query.mock.calls.filter(([sql]) => String(sql).startsWith("WITH sampled"))).toHaveLength(2);
-});
-
-test("fails explicitly when both randomized and sequential sample queries time out", async () => {
-  const note = column("33333333-3333-4333-8333-333333333333", "note");
-  const query = vi.fn(async (sql: string) => {
-    if (sql.startsWith("WITH sampled")) {
-      throw Object.assign(new Error("raw source detail"), { code: "57014" });
-    }
-    return { rows: [] };
-  });
-  const access: CatalogPostgresAccess = {
-    connect: vi.fn(async () => ({ query, end: vi.fn(async () => undefined) }) as CatalogDatabaseClient),
-  };
-
-  await expect(new ConcreteSensitivityValueSource(access).scanTable(
-    request([note], { fullScanThreshold: undefined }),
-    vi.fn(),
-    new AbortController().signal,
-  )).rejects.toEqual(new CatalogConnectorError("Sensitivity sample query timed out"));
-});

backend/test/catalog-tables-routes.test.ts

@@ -13,11 +13,16 @@ const roots: string[] = [];
 afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
 
 const workspace: WorkspaceDescriptor = {
-  workspace: { schema_version: 4, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
+  workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
   dwh: {
     engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432,
     supported_transports: ["postgres_direct", "rest_api"],
   },
+  semantic_index: {
+    vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
+    embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
+  },
+  llm_policy: { allowed: ["zai/glm-5.2"] },
   diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } },
 };
 const revision: WorkspaceRevision = {
@@ -103,7 +108,7 @@ test("requires an exact deletion confirmation before applying the atomic diff",
   ]);
 });
 
-test("starts synchronization without requiring a prior connection test", async () => {
+test("refuses synchronization until the current binding has passed its connection test", async () => {
   const { app, repository, database } = await setup();
   await repository.update(database.id, database.version, {
     workspaceId: database.workspaceId,
@@ -117,10 +122,6 @@ test("starts synchronization without requiring a prior connection test", async (
     url: `/catalog/databases/${database.id}/sync-runs`,
     payload: { version: database.version + 1, scope: "tables", tableIds: [] },
   });
-  expect(response.statusCode).toBe(202);
-  expect(response.json()).toMatchObject({
-    databaseId: database.id,
-    scope: "tables",
-    state: "queued",
-  });
+  expect(response.statusCode).toBe(409);
+  expect(response.json()).toMatchObject({ code: "schema_sync_conflict" });
 });

backend/test/config.test.ts

@@ -71,7 +71,6 @@ test("loadConfig keeps local development defaults", () => {
     workspaceSecretRuntimeRoot: "/tmp/thothii-workspace-secrets",
     internalQdrantUrl: "http://qdrant:6333",
     internalEmbeddingUrl: "http://embedding:11434",
-    internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
     internalEmbeddingModel: "qwen3-embedding:0.6b",
     internalEmbeddingDimensions: 1024,
     authMode: "none",
@@ -81,35 +80,6 @@ test("loadConfig keeps local development defaults", () => {
   expect(loadConfig({}).dataRoot).toBeUndefined();
 });
 
-test("loadConfig keeps local NER disabled unless an absolute model path is configured", () => {
-  expect(loadConfig({}).sensitivityNer).toBeUndefined();
-  expect(loadConfig({
-    THT_SENSITIVITY_NER_MODEL_PATH: "/models/gliner2-pii",
-  }).sensitivityNer?.pythonExecutable).toBe("/opt/sensitivity-ner/bin/python");
-  expect(loadConfig({
-    THT_SENSITIVITY_NER_MODEL_PATH: "/models/gliner2-pii",
-    THT_SENSITIVITY_NER_PYTHON: "/opt/sensitivity-ner/bin/python",
-    THT_SENSITIVITY_NER_WORKER: "/app/backend/python/sensitivity_ner_worker.py",
-    THT_SENSITIVITY_NER_THREADS: "3",
-  }).sensitivityNer).toEqual({
-    modelPath: "/models/gliner2-pii",
-    pythonExecutable: "/opt/sensitivity-ner/bin/python",
-    workerScript: "/app/backend/python/sensitivity_ner_worker.py",
-    threads: 3,
-  });
-});
-
-test("loadConfig rejects ambiguous or unsafe local NER configuration", () => {
-  expect(() => loadConfig({ THT_SENSITIVITY_NER_MODEL_PATH: "fastino/model" }))
-    .toThrow("sensitivity NER model path configuration is invalid");
-  expect(() => loadConfig({
-    THT_SENSITIVITY_NER_MODEL_PATH: "/models/gliner2-pii",
-    THT_SENSITIVITY_NER_THREADS: "0",
-  })).toThrow("sensitivity NER thread configuration is invalid");
-  expect(() => loadConfig({ THT_SENSITIVITY_NER_PYTHON: "/opt/ner/bin/python" }))
-    .toThrow("sensitivity NER settings require a model path");
-});
-
 test("loadConfig allows none and mock only outside production when auth.yaml is absent", () => {
   const originalNodeEnvironment = process.env.NODE_ENV;
   delete process.env.NODE_ENV;
@@ -228,22 +198,6 @@ test("loadConfig accepts only the allowed internal semantic runtime hosts", () =
     .toThrow(/internal.*embedding|invalid/i);
 });
 
-test("loadConfig derives the embedding runtime model from its canonical catalog identity", () => {
-  expect(loadConfig({
-    THT_INTERNAL_EMBEDDING_ID: "ollama/nomic-embed-text",
-    THT_INTERNAL_EMBEDDING_MODEL: "nomic-embed-text",
-  })).toMatchObject({
-    internalEmbeddingId: "ollama/nomic-embed-text",
-    internalEmbeddingModel: "nomic-embed-text",
-  });
-  expect(() => loadConfig({
-    THT_INTERNAL_EMBEDDING_ID: "ollama/nomic-embed-text",
-    THT_INTERNAL_EMBEDDING_MODEL: "different-model",
-  })).toThrow("does not match its canonical identity");
-  expect(() => loadConfig({ THT_INTERNAL_EMBEDDING_ID: "not-canonical" }))
-    .toThrow("embedding identity configuration is invalid");
-});
-
 test("loadConfig enables the legacy workspace request only through explicit local mode", () => {
   expect(loadConfig({ THT_LEGACY_WORKSPACE_MODE: "local" }).legacyWorkspaceMode).toBe(true);
 

backend/test/effective-config.test.ts

@@ -11,7 +11,6 @@ import {
 const semanticRuntime = {
   internalQdrantUrl: "http://qdrant:6333",
   internalEmbeddingUrl: "http://embedding:11434",
-  internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
   internalEmbeddingModel: "qwen3-embedding:0.6b",
   internalEmbeddingDimensions: 1024,
 };
@@ -41,10 +40,7 @@ function directRendered(): Record<string, unknown> {
       vector: {
         engine: "qdrant",
         base_url: "http://qdrant:6333",
-        collections: {
-          reference: "psd-clinical-reference",
-          memory: "psd-clinical-memory",
-        },
+        collection: "psd-clinical",
         dimensions: 1024,
         distance: "cosine",
         collection_lifecycle: "require_existing",
@@ -106,11 +102,11 @@ function restRendered(): Record<string, unknown> {
 }
 
 const directCanonical =
-  `{"schemaVersion":3,"dwh":{` +
+  `{"schemaVersion":1,"dwh":{` +
   `"engine":"postgres","database":"postgres","schema":"datawarehouse",` +
   `"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` +
-  `"vector":{"collections":{"reference":"psd-clinical-reference","memory":"psd-clinical-memory"},"dimensions":1024,"distance":"cosine"},` +
-  `"embedding":{"id":"ollama/qwen3-embedding:0.6b","model":"qwen3-embedding:0.6b","dimensions":1024},` +
+  `"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` +
+  `"embedding":{"model":"qwen3-embedding:0.6b","dimensions":1024},` +
   `"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` +
   `"indexes":"/data/sessions/psd-clinical/indexes"}}`;
 
@@ -131,7 +127,7 @@ test("canonical effective config excludes secrets, evidence, session storage, an
   expect(json).not.toContain("sources");
   expect(json).not.toContain("collection_lifecycle");
   expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity
-  expect(json).not.toContain('"memory":"/data');
+  expect(json).not.toContain("memory");
   expect(json).not.toContain('"sessions"');
 });
 
@@ -193,21 +189,9 @@ test("DWH-affecting changes alter the effective config identity", () => {
   const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } };
   expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore);
 
-  const changedCollection = {
-    ...base,
-    resources: {
-      ...base.resources,
-      vector: {
-        ...(base.resources as Record<string, any>).vector,
-        collections: { reference: "other-reference", memory: "other-memory" },
-      },
-    },
-  };
+  const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record<string, any>).vector, collection: "other" } } };
   expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore);
 
-  const changedEmbeddingIdentity = { ...base, resources: { ...base.resources, embeddings: { ...(base.resources as Record<string, any>).embeddings, model: "other-embedding" } } };
-  expect(effectiveConfigIdentity("psd-clinical", changedEmbeddingIdentity)).not.toBe(identityBefore);
-
   const changedTransport = restRendered();
   expect(effectiveConfigIdentity("psd-clinical", changedTransport)).not.toBe(identityBefore);
 });

backend/test/list-models.test.ts

@@ -6,7 +6,6 @@ import { join } from "node:path";
 import path from "node:path";
 import { createPiModelLister } from "../src/pi/list-models.js";
 import { loadConfig } from "../src/config.js";
-import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
 
 const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
 
@@ -45,36 +44,6 @@ test("createPiModelLister returns mapped PiModel[] from get_available_models", a
   }
 });
 
-test("catalog listing translates upstream Pi IDs back to canonical model keys", async () => {
-  const script = scriptWith([
-    { provider: "local", id: "qwen2.5:7b", name: "Upstream label", reasoning: false },
-  ]);
-  const model: RuntimeModel = {
-    id: "local/qwen", provider: "local", model: "qwen", label: "Catalog Qwen",
-    upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
-    authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
-    session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 },
-  };
-  const modelCatalog: RuntimeModelCatalog = {
-    defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
-    sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
-  };
-  try {
-    const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
-      ...noManagedModels,
-      modelCatalog,
-      loadEnabledModels: enabled("local/qwen2.5:7b"),
-      spawnFn: () => spawn("node", [FAKE, script]) as any,
-    });
-
-    await expect(lister()).resolves.toEqual([{
-      provider: "local", id: "qwen", name: "Catalog Qwen", reasoning: true,
-    }]);
-  } finally {
-    rmSync(path.dirname(script), { recursive: true, force: true });
-  }
-});
-
 test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
   const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
   try {

backend/test/metadata-generation-models.test.ts

@@ -1,12 +1,15 @@
 import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterEach, expect, test } from "vitest";
+import { afterEach, expect, test, vi } from "vitest";
+import { buildApp } from "../src/app.js";
+import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
 import {
   loadMetadataGenerationModels,
   MetadataGenerationModelUnavailableError,
 } from "../src/catalog/metadata-generation-models.js";
-import { loadRuntimeModelCatalog, splitCanonicalModelId } from "../src/models/runtime-model-catalog.js";
+import { loadConfig } from "../src/config.js";
+import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
 
 const roots: string[] = [];
 
@@ -14,101 +17,260 @@ afterEach(() => {
   for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
 });
 
-function runtimeCatalog(overrides: Record<string, unknown> = {}, secrets = "OPENAI_API_KEY=raw-provider-secret\n") {
-  const root = mkdtempSync(join(tmpdir(), "thothii-runtime-models-"));
+function metadataConfiguration(
+  metadataGeneration: string,
+  secrets = "OPENAI_API_KEY=raw-provider-secret\n",
+) {
+  const root = mkdtempSync(join(tmpdir(), "thothii-metadata-models-"));
   roots.push(root);
-  const catalogFile = join(root, "catalog.json");
+  const installationFile = join(root, "thothii-installation.yaml");
   const secretsFile = join(root, "thothii.secrets");
-  const catalog = {
-    schemaVersion: 1,
-    defaultSession: "zai/glm-5.3",
-    defaultMetadataGeneration: "zai/glm-5.3",
-    embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
-    models: [
-      {
-        id: "zai/glm-5.3", provider: "zai", model: "glm-5.3", label: "GLM 5.3",
-        upstreamModel: "glm-5.3", endpoint: { baseUrl: "https://api.z.ai/v1" },
-        authentication: { mode: "secret_env", apiKeyEnv: "OPENAI_API_KEY" },
-        sessionAdapter: { mode: "openai_compatible" },
-        metadataAdapter: { litellmProvider: "openai" },
-        session: { reasoning: true, contextWindow: 200000, maxTokens: 131072 },
-        metadataGeneration: { disableThinking: false },
-      },
-      {
-        id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
-        label: "DeepSeek V4 Pro", upstreamModel: "deepseek-v4-pro",
-        authentication: { mode: "pi_auth" }, sessionAdapter: { mode: "pi_builtin" },
-        session: { reasoning: false },
-      },
-    ],
-    ...overrides,
-  };
-  writeFileSync(catalogFile, JSON.stringify(catalog), { mode: 0o600 });
+  writeFileSync(installationFile, metadataGeneration, { mode: 0o600 });
   writeFileSync(secretsFile, secrets, { mode: 0o600 });
-  chmodSync(catalogFile, 0o600);
+  chmodSync(installationFile, 0o600);
   chmodSync(secretsFile, 0o600);
-  return { catalogFile, secretsFile };
+  return { installationFile, secretsFile };
 }
 
-test("loads session default and safe metadata choices from the normalized runtime catalog", () => {
-  const { catalogFile, secretsFile } = runtimeCatalog();
-  const runtime = loadRuntimeModelCatalog(catalogFile);
-  const metadata = loadMetadataGenerationModels({ catalogFile, secretsFile });
-
-  expect(runtime.defaultSession).toBe("zai/glm-5.3");
-  expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(true);
-  expect(metadata.catalog()).toEqual({
-    models: [{ id: "zai/glm-5.3", label: "GLM 5.3" }],
-    default: "zai/glm-5.3",
+function appFor(installationFile: string, secretsFile: string) {
+  const config = loadConfig({
+    NODE_ENV: "test",
+    THT_HARNESS_DIR: "/missing",
+    THT_INSTALLATION_CONFIG_FILE: installationFile,
+    THT_SECRETS_FILE: secretsFile,
+    PI_PROVIDER: "unrelated-pi-provider",
+    PI_MODEL: "unrelated-pi-model",
   });
-  expect(metadata.resolve("zai/glm-5.3")).toEqual({
-    id: "zai/glm-5.3", provider: "openai", model: "glm-5.3",
-    endpoint: { baseUrl: "https://api.z.ai/v1" },
-    apiKeyEnv: "OPENAI_API_KEY", apiKey: "raw-provider-secret",
+  return buildApp(config, {
+    thtRunner: {} as never,
+    workspaceRegistry: { list: vi.fn(async () => []) } as unknown as WorkspaceRegistry,
+    workspaceDiagnoser: vi.fn(),
+    catalogRepository: new MemoryCatalogRepository(),
   });
-  expect(() => metadata.resolve("zai/missing")).toThrow(MetadataGenerationModelUnavailableError);
-});
+}
 
-test("returns empty catalogs when no runtime projection is configured", () => {
-  expect(loadRuntimeModelCatalog().defaultSession).toBeNull();
-  expect(loadMetadataGenerationModels({}).catalog()).toEqual({ models: [], default: null });
-});
+test("exposes only safe metadata-generation choices and their configured default", async () => {
+  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
+  default: openai-mini
+  models:
+    - id: openai-mini
+      label: OpenAI Mini
+      litellm:
+        provider: openai
+        model: gpt-4.1-mini
+        endpoint:
+          baseUrl: https://api.openai.example/v1
+          apiVersion: "2026-08-01"
+      apiKeyEnv: OPENAI_API_KEY
+`);
+  const app = appFor(installationFile, secretsFile);
 
-test("rejects a drifted default and an unprotected projection", () => {
-  const drifted = runtimeCatalog({ defaultSession: "zai/missing" });
-  expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("session default is invalid");
+  const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" });
 
-  const unprotected = runtimeCatalog();
-  chmodSync(unprotected.catalogFile, 0o666);
-  expect(() => loadRuntimeModelCatalog(unprotected.catalogFile)).toThrow("runtime model catalog is unavailable");
-});
-
-test("rejects authentication semantics that cannot come from the installation catalog", () => {
-  const invalid = runtimeCatalog({
-    defaultMetadataGeneration: undefined,
-    models: [{
-      id: "zai/glm-5.3",
-      provider: "zai",
-      model: "glm-5.3",
-      label: "GLM 5.3",
-      upstreamModel: "glm-5.3",
-      authentication: { mode: "secret_env" },
-      sessionAdapter: { mode: "pi_builtin" },
-      session: { reasoning: true },
-    }],
+  expect(response.statusCode).toBe(200);
+  expect(response.json()).toEqual({
+    models: [{ id: "openai-mini", label: "OpenAI Mini" }],
+    default: "openai-mini",
   });
-  expect(() => loadRuntimeModelCatalog(invalid.catalogFile)).toThrow("runtime model catalog is invalid");
+  expect(response.body).not.toMatch(/openai\/gpt|gpt-4\.1|api\.openai|OPENAI_API_KEY|raw-provider-secret/);
+  await app.close();
 });
 
-test("fails closed for missing or unusable provider secrets", () => {
-  const missing = runtimeCatalog({}, "THT_DWH_API_KEY=other\n");
-  expect(() => loadMetadataGenerationModels(missing)).toThrow('secret "OPENAI_API_KEY" is missing');
+test("rejects an unprotected installation descriptor", () => {
+  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
+  default: openai-mini
+  models:
+    - id: openai-mini
+      label: OpenAI Mini
+      litellm: {provider: openai, model: gpt-4.1-mini}
+      apiKeyEnv: OPENAI_API_KEY
+`);
+  chmodSync(installationFile, 0o644);
 
-  const unusable = runtimeCatalog({}, "OPENAI_API_KEY=contains whitespace\n");
-  expect(() => loadMetadataGenerationModels(unusable)).toThrow('secret "OPENAI_API_KEY" is unusable');
+  expect(() => loadMetadataGenerationModels({ installationFile, secretsFile }))
+    .toThrow("metadata-generation installation is unavailable");
 });
 
-test("splits canonical session identities without provider aliases", () => {
-  expect(splitCanonicalModelId("zai/glm-5.3")).toEqual({ provider: "zai", model: "glm-5.3" });
-  expect(() => splitCanonicalModelId("glm-5.3")).toThrow("model identity is invalid");
+test("returns an empty safe catalog when no metadata-generation model is configured", async () => {
+  const { installationFile, secretsFile } = metadataConfiguration("profile: local\n");
+  const app = appFor(installationFile, secretsFile);
+
+  const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" });
+
+  expect(response.statusCode).toBe(200);
+  expect(response.json()).toEqual({ models: [], default: null });
+  await app.close();
+});
+
+test("resolves only a configured selection for the later generation boundary", () => {
+  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
+  default: openai-mini
+  models:
+    - id: openai-mini
+      label: OpenAI Mini
+      litellm:
+        provider: openai
+        model: gpt-4.1-mini
+        endpoint: {baseUrl: https://api.openai.example/v1, apiVersion: "2026-08-01"}
+      apiKeyEnv: OPENAI_API_KEY
+`);
+  const models = loadMetadataGenerationModels({ installationFile, secretsFile });
+
+  expect(models.resolve("openai-mini")).toEqual({
+    id: "openai-mini",
+    provider: "openai",
+    model: "gpt-4.1-mini",
+    endpoint: { baseUrl: "https://api.openai.example/v1", apiVersion: "2026-08-01" },
+    apiKeyEnv: "OPENAI_API_KEY",
+    apiKey: "raw-provider-secret",
+  });
+  expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError);
+});
+
+test("loads DeepSeek models, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
+  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
+  default: glm-53
+  models:
+    - id: deepseek-v4-pro
+      label: DeepSeek V4 Pro
+      litellm: {provider: deepseek, model: deepseek-v4-pro}
+      apiKeyEnv: DEEPSEEK_API_KEY
+    - id: deepseek-v4-flash
+      label: DeepSeek V4 Flash
+      litellm: {provider: deepseek, model: deepseek-v4-flash}
+      apiKeyEnv: DEEPSEEK_API_KEY
+    - id: glm-53
+      label: GLM 5.3
+      litellm:
+        provider: openai
+        model: glm-5.3
+        endpoint: {baseUrl: https://api.z.ai/api/coding/paas/v4}
+      apiKeyEnv: ZAI_API_KEY
+    - id: qwen-36
+      label: Qwen 3.6
+      litellm:
+        provider: openai
+        model: qwen3.6-35b-a3b
+        disableThinking: true
+        endpoint: {baseUrl: https://models.internal.example/v1}
+`, "DEEPSEEK_API_KEY=deepseek-secret\nZAI_API_KEY=zai-secret\n");
+
+  const models = loadMetadataGenerationModels({ installationFile, secretsFile });
+
+  expect(models.catalog()).toEqual({
+    models: [
+      { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
+      { id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
+      { id: "glm-53", label: "GLM 5.3" },
+      { id: "qwen-36", label: "Qwen 3.6" },
+    ],
+    default: "glm-53",
+  });
+  expect(models.resolve("deepseek-v4-pro")).toMatchObject({
+    apiKeyEnv: "DEEPSEEK_API_KEY",
+    apiKey: "deepseek-secret",
+  });
+  expect(models.resolve("qwen-36")).toEqual({
+    id: "qwen-36",
+    provider: "openai",
+    model: "qwen3.6-35b-a3b",
+    disableThinking: true,
+    endpoint: { baseUrl: "https://models.internal.example/v1" },
+  });
+});
+
+test("loads an explicit keyless endpoint without a secret bundle", () => {
+  const { installationFile } = metadataConfiguration(`metadataGeneration:
+  default: qwen-36
+  models:
+    - id: qwen-36
+      label: Qwen 3.6
+      litellm:
+        provider: openai
+        model: qwen3.6-35b-a3b
+        disableThinking: true
+        endpoint: {baseUrl: https://models.internal.example/v1}
+`);
+
+  expect(loadMetadataGenerationModels({ installationFile }).resolve("qwen-36")).toEqual({
+    id: "qwen-36",
+    provider: "openai",
+    model: "qwen3.6-35b-a3b",
+    disableThinking: true,
+    endpoint: { baseUrl: "https://models.internal.example/v1" },
+  });
+});
+
+test.each([
+  ["invalid YAML", "metadataGeneration: [\n", "OPENAI_API_KEY=secret\n", /invalid YAML/],
+  ["duplicate ids", `metadataGeneration:
+  default: openai-mini
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
+    - {id: openai-mini, label: Two, litellm: {provider: openai, model: gpt-4.1}, apiKeyEnv: OPENAI_API_KEY}
+`, "OPENAI_API_KEY=secret\n", /model id "openai-mini" is duplicated/],
+  ["missing default", `metadataGeneration:
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
+`, "OPENAI_API_KEY=secret\n", /default is required/],
+  ["unknown default", `metadataGeneration:
+  default: absent
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
+`, "OPENAI_API_KEY=secret\n", /default "absent" is not configured/],
+  ["malformed settings", `metadataGeneration:
+  default: openai-mini
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: "open ai", model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
+`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
+  ["malformed endpoint", `metadataGeneration:
+  default: openai-mini
+  models:
+    - id: openai-mini
+      label: One
+      litellm: {provider: openai, model: gpt-4.1-mini, endpoint: {baseUrl: not-a-url}}
+      apiKeyEnv: OPENAI_API_KEY
+`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
+  ["keyless hosted model without endpoint", `metadataGeneration:
+  default: openai-mini
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}}
+`, "", /configuration is invalid/],
+  ["disable thinking without endpoint", `metadataGeneration:
+  default: openai-mini
+  models:
+    - id: openai-mini
+      label: One
+      litellm: {provider: openai, model: gpt-4.1-mini, disableThinking: true}
+      apiKeyEnv: OPENAI_API_KEY
+`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
+  ["unallowed secret reference", `metadataGeneration:
+  default: openai-mini
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: THT_DWH_API_KEY}
+`, "THT_DWH_API_KEY=secret\n", /configuration is invalid/],
+  ["missing referenced secret", `metadataGeneration:
+  default: openai-mini
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
+`, "THT_DWH_API_KEY=secret\n", /secret "OPENAI_API_KEY" is missing/],
+  ["unusable referenced secret", `metadataGeneration:
+  default: openai-mini
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
+`, "OPENAI_API_KEY=secret with whitespace\n", /secret "OPENAI_API_KEY" is unusable/],
+] as const)("rejects %s metadata-generation configuration", (_name, yaml, secrets, expected) => {
+  const { installationFile, secretsFile } = metadataConfiguration(yaml, secrets);
+  expect(() => loadMetadataGenerationModels({ installationFile, secretsFile })).toThrow(expected);
+});
+
+test("rejects a missing secret-bundle declaration for configured models", () => {
+  const { installationFile } = metadataConfiguration(`metadataGeneration:
+  default: openai-mini
+  models:
+    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
+`);
+
+  expect(() => loadMetadataGenerationModels({ installationFile }))
+    .toThrow("metadata-generation keyed models require THT_SECRETS_FILE");
 });

backend/test/pi-management.test.ts

@@ -1,13 +1,13 @@
-import { mkdtempSync } from "node:fs";
+import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { expect, test, vi } from "vitest";
 import { loadConfig } from "../src/config.js";
 import {
+  PiManagementError,
   createPiManagement,
   type PiExecFile,
 } from "../src/pi/management.js";
-import type { RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
 
 function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-management-")), "settings.json")) {
   return loadConfig({
@@ -18,14 +18,10 @@ function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-manage
   });
 }
 
-const modelCatalog: RuntimeModelCatalog = {
-  defaultSession: "zai/glm-5.2",
-  defaultMetadataGeneration: null,
-  embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
-  sessionModels: () => [],
-  metadataModels: () => [],
-  hasSession: (id) => id === "zai/glm-5.2",
-};
+const supportedModels = [
+  { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true },
+  { provider: "deepseek", id: "deepseek-v4", name: "DeepSeek V4", reasoning: true },
+];
 
 function successfulExec(calls: Array<{ command: string; args: string[]; timeout: number }>): PiExecFile {
   return async (command, args, options) => {
@@ -40,7 +36,7 @@ test("status parses only a Pi version from a fixed execFile argument array", asy
   const calls: Array<{ command: string; args: string[]; timeout: number }> = [];
   const service = createPiManagement(configFor(), {
     execute: successfulExec(calls),
-    modelCatalog,
+    listModels: async () => supportedModels,
     readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
     credentialStatus: () => "missing",
     now: () => new Date("2026-08-05T10:00:00.000Z"),
@@ -67,7 +63,7 @@ test.each(["present", "missing"] as const)(
     const checkedProviders: Array<string | undefined> = [];
     const service = createPiManagement(configFor(), {
       execute: successfulExec([]),
-      modelCatalog,
+      listModels: async () => supportedModels,
       readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
       credentialStatus: (provider) => {
         checkedProviders.push(provider);
@@ -89,6 +85,100 @@ test.each(["present", "missing"] as const)(
   },
 );
 
+// Catches an options response that leaks provider metadata or lets callers choose model IDs that
+// Pi did not explicitly enable for this installation.
+test("options expose only closed provider, model, and reasoning choices", async () => {
+  const service = createPiManagement(configFor(), {
+    execute: successfulExec([]),
+    listModels: async () => supportedModels,
+    now: () => new Date("2026-08-05T10:00:00.000Z"),
+  });
+
+  await expect(service.options()).resolves.toEqual({
+    providers: ["zai", "deepseek"],
+    models: [
+      { provider: "zai", id: "glm-5.2" },
+      { provider: "deepseek", id: "deepseek-v4" },
+    ],
+    reasoning: ["low", "medium", "high"],
+    checkedAt: "2026-08-05T10:00:00.000Z",
+  });
+});
+
+// Catches raw managed models.json validation details being collapsed into an ambiguous model-list
+// failure or escaping through the Pi Management options API.
+test("options report invalid managed model configuration with a stable sanitized error", async () => {
+  const service = createPiManagement(configFor(), {
+    execute: successfulExec([]),
+    listModels: async () => {
+      throw Object.assign(
+        new Error("!sensitive-command /private/models.json raw-secret"),
+        { code: "PI_MANAGED_CONFIG_INVALID" },
+      );
+    },
+  });
+
+  let caught: unknown;
+  try {
+    await service.options();
+  } catch (error) {
+    caught = error;
+  }
+  expect(caught).toMatchObject<PiManagementError>({
+    code: "pi_management_unavailable",
+    message: "Pi provider/model configuration is invalid",
+  });
+  expect(String(caught)).not.toMatch(/sensitive|private|models\.json|secret/i);
+});
+
+// Catches configuration writes that accept whitespace, unknown choices, or extra free-form fields
+// before reaching the durable installation settings file.
+test("config rejects invalid free-form values before writing settings", async () => {
+  const directory = mkdtempSync(join(tmpdir(), "tht-pi-management-invalid-"));
+  try {
+    let writes = 0;
+    const service = createPiManagement(configFor(join(directory, "settings.json")), {
+      execute: successfulExec([]),
+      listModels: async () => supportedModels,
+      readSettings: () => ({}),
+      saveSettings: () => { writes += 1; return {}; },
+    });
+
+    await expect(service.configure({
+      provider: "zai ", model: "glm-5.2", reasoning: "medium", unexpected: "value",
+    } as any)).rejects.toMatchObject<PiManagementError>({ code: "pi_management_invalid_config" });
+    expect(writes).toBe(0);
+  } finally {
+    rmSync(directory, { recursive: true, force: true });
+  }
+});
+
+// Catches a non-atomic implementation that can leave partial settings or temporary files after a
+// normal installation-default update.
+test("config validates closed choices and atomically persists non-secret defaults", async () => {
+  const directory = mkdtempSync(join(tmpdir(), "tht-pi-management-write-"));
+  const settingsFile = join(directory, "settings.json");
+  try {
+    const service = createPiManagement(configFor(settingsFile), {
+      execute: successfulExec([]),
+      listModels: async () => supportedModels,
+      now: () => new Date("2026-08-05T10:00:00.000Z"),
+    });
+
+    await expect(service.configure({
+      provider: "zai", model: "glm-5.2", reasoning: "high",
+    })).resolves.toEqual({
+      provider: "zai", model: "glm-5.2", reasoning: "high", updatedAt: "2026-08-05T10:00:00.000Z",
+    });
+    expect(JSON.parse(readFileSync(settingsFile, "utf8"))).toEqual({
+      provider: "zai", model: "glm-5.2", thinking: "high",
+    });
+    expect(readdirSync(directory)).toEqual(["settings.json"]);
+  } finally {
+    rmSync(directory, { recursive: true, force: true });
+  }
+});
+
 // Catches a hung Pi smoke check that leaves an operator waiting indefinitely or returns raw child
 // diagnostics containing provider credentials.
 test("smoke uses the configured timeout and reports a sanitized timeout", async () => {
@@ -98,7 +188,7 @@ test("smoke uses the configured timeout and reports a sanitized timeout", async
       calls.push({ command, args, timeout: options.timeout });
       throw Object.assign(new Error("provider token=raw-provider-token"), { code: "ETIMEDOUT" });
     },
-    modelCatalog,
+    listModels: async () => supportedModels,
     readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
     now: () => new Date("2026-08-05T10:00:00.000Z"),
   });
@@ -120,7 +210,7 @@ test("smoke exercises the configured provider and model", async () => {
   const providerChecks: unknown[] = [];
   const service = createPiManagement(configFor(), {
     execute: successfulExec([]),
-    modelCatalog,
+    listModels: async () => supportedModels,
     readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
     smokeProvider: async (request) => { providerChecks.push(request); },
     now: () => new Date("2026-08-05T10:00:00.000Z"),
@@ -140,7 +230,7 @@ test("smoke exercises the configured provider and model", async () => {
 test("smoke fails closed and sanitizes configured-provider authentication errors", async () => {
   const service = createPiManagement(configFor(), {
     execute: successfulExec([]),
-    modelCatalog,
+    listModels: async () => supportedModels,
     readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
     smokeProvider: async () => {
       throw new Error('401 {"token":"raw-expired-token","output":"raw-provider-output"}');
@@ -162,7 +252,7 @@ test("smoke fails closed and sanitizes configured-provider authentication errors
 test("smoke reports invalid managed provider configuration with a stable sanitized error", async () => {
   const service = createPiManagement(configFor(), {
     execute: successfulExec([]),
-    modelCatalog,
+    listModels: async () => supportedModels,
     readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
     smokeProvider: async () => {
       throw Object.assign(
@@ -194,7 +284,7 @@ test("smoke applies one deadline across version and a hung provider turn", async
         () => resolve({ stdout: "pi 0.80.3\n", stderr: "" }),
         500,
       )),
-      modelCatalog,
+      listModels: async () => supportedModels,
       readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
       smokeProvider: async ({ timeoutMs }) => {
         providerTimeouts.push(timeoutMs);
@@ -230,7 +320,7 @@ test("logs keep only the latest 200 redacted lines", async () => {
   source[201] = "THT_MODEL_API_KEY=raw-env-secret";
   const service = createPiManagement(configFor(), {
     execute: successfulExec([]),
-    modelCatalog,
+    listModels: async () => supportedModels,
     readLogs: () => source.join("\n"),
     now: () => new Date("2026-08-05T10:00:00.000Z"),
   });

backend/test/pi-process-manager.test.ts

@@ -8,7 +8,6 @@ import {
 } from "node:fs";
 import { tmpdir } from "node:os";
 import { PiProcessManager } from "../src/pi/pi-process-manager.js";
-import type { RuntimeModelCatalog, RuntimeModel } from "../src/models/runtime-model-catalog.js";
 import { loadConfig } from "../src/config.js";
 import {
   PI_MANAGED_CONFIG_ERROR_MESSAGE,
@@ -642,53 +641,6 @@ test("session Pi spawn reads the single secret bundle and scrubs its path", asyn
   }
 });
 
-test("session Pi spawn resolves the selected catalog credential from the secret bundle", () => {
-  const root = mkdtempSync(path.join(tmpdir(), "thothii-catalog-credential-"));
-  const agentDir = path.join(root, "agent");
-  mkdirSync(agentDir, { mode: 0o700 });
-  writeFileSync(path.join(agentDir, "auth.json"), "{}\n", { mode: 0o600 });
-  writeFileSync(path.join(agentDir, "models.json"), '{"providers":{}}\n', { mode: 0o600 });
-  const secret = path.join(root, "thothii.secrets");
-  writeFileSync(secret, "ZAI_API_KEY=catalog-secret\nTHT_MODEL_API_KEY=legacy-secret\n", { mode: 0o600 });
-  const model: RuntimeModel = {
-    id: "openai/test-model",
-    provider: "openai",
-    model: "test-model",
-    label: "Test model",
-    upstreamModel: "test-model",
-    authentication: { mode: "secret_env", apiKeyEnv: "ZAI_API_KEY" },
-    sessionAdapter: { mode: "pi_builtin" },
-    session: { reasoning: false },
-  };
-  const modelCatalog: RuntimeModelCatalog = {
-    defaultSession: model.id,
-    defaultMetadataGeneration: null,
-    embedding: null,
-    sessionModels: () => [model],
-    metadataModels: () => [],
-    hasSession: (id) => id === model.id,
-  };
-  const calls: any[][] = [];
-  const child = recordingChild();
-  child.stderr.resume = () => {};
-  vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
-  const mgr = new PiProcessManager(loadConfig({ THT_SECRETS_FILE: secret }), {
-    modelCatalog,
-    authProviders: () => new Set(),
-    spawnFn: (...args: any[]) => { calls.push(args); return child as any; },
-  });
-  try {
-    mgr.createFor("catalog-credential", { provider: "openai", model: "test-model" });
-    expect(calls[0][2].env.ZAI_API_KEY).toBe("catalog-secret");
-    expect(calls[0][2].env).not.toHaveProperty("OPENAI_API_KEY");
-    expect(calls[0][2].env).not.toHaveProperty("THT_MODEL_API_KEY");
-  } finally {
-    mgr.teardown("catalog-credential");
-    vi.unstubAllEnvs();
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
 test.each([["OpenAI", "openai"], ["gemini", "google"]])(
   "set_model uses canonical packaged provider ID for %s", async (provider, canonical) => {
     const secret = path.resolve(__dirname, `.canonical-key-${process.pid}-${provider}`);
@@ -718,54 +670,6 @@ test.each([["OpenAI", "openai"], ["gemini", "google"]])(
   },
 );
 
-test("set_model translates a canonical catalog key to its upstream Pi model ID", async () => {
-  const root = mkdtempSync(path.join(tmpdir(), "thothii-upstream-model-"));
-  const agentDir = path.join(root, "agent");
-  mkdirSync(agentDir, { mode: 0o700 });
-  writeFileSync(path.join(agentDir, "models.json"), JSON.stringify({
-    providers: {
-      local: {
-        baseUrl: "http://ollama:11434/v1", apiKey: "local",
-        models: [{ id: "qwen2.5:7b" }],
-      },
-    },
-  }), { mode: 0o600 });
-  vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
-  const child = recordingChild();
-  child.stderr.resume = () => {};
-  child.stdin.write = (data: unknown) => {
-    const request = JSON.parse(String(data));
-    child._writes.push(String(data));
-    if (request.id) {
-      queueMicrotask(() => child.stdout.emit("data", `${JSON.stringify({
-        type: "response", id: request.id, success: true,
-      })}\n`));
-    }
-    return true;
-  };
-  const model: RuntimeModel = {
-    id: "local/qwen", provider: "local", model: "qwen", label: "Qwen",
-    upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
-    authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
-    session: { reasoning: false, contextWindow: 32768, maxTokens: 8192 },
-  };
-  const modelCatalog: RuntimeModelCatalog = {
-    defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
-    sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
-  };
-  const mgr = new PiProcessManager(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
-    modelCatalog, authProviders: () => new Set(), spawnFn: () => child as any,
-  });
-  try {
-    await mgr.spawnFor("upstream-model", { provider: "local", model: "qwen" });
-    expect(child._writes.join("")).toContain('"modelId":"qwen2.5:7b"');
-  } finally {
-    mgr.teardown("upstream-model");
-    vi.unstubAllEnvs();
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
 test.each(["installation-local", "private-compatible"])(
   "provider %s configured with a literal apiKey spawns without a managed key",
   async (provider) => {

backend/test/pi-provider-smoke.test.ts

@@ -1,11 +1,9 @@
 import { EventEmitter } from "node:events";
-import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { dirname } from "node:path";
 import { afterEach, expect, test, vi } from "vitest";
 import { loadConfig } from "../src/config.js";
 import { createPiProviderSmoke } from "../src/pi/provider-smoke.js";
-import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
 
 afterEach(() => vi.unstubAllEnvs());
 
@@ -46,50 +44,6 @@ function successfulProviderChild() {
 
 const MANAGED_CONFIG_ERROR = "Pi provider/model configuration is invalid";
 
-test("provider smoke resolves the selected catalog credential from the secret bundle", async () => {
-  const root = mkdtempSync(join(tmpdir(), "thothii-smoke-catalog-credential-"));
-  const secret = join(root, "thothii.secrets");
-  writeFileSync(secret, "ZAI_API_KEY=catalog-secret\nTHT_MODEL_API_KEY=legacy-secret\n", { mode: 0o600 });
-  const model: RuntimeModel = {
-    id: "openai/test-model",
-    provider: "openai",
-    model: "test-model",
-    label: "Test model",
-    upstreamModel: "test-model",
-    authentication: { mode: "secret_env", apiKeyEnv: "ZAI_API_KEY" },
-    sessionAdapter: { mode: "pi_builtin" },
-    session: { reasoning: false },
-  };
-  const modelCatalog: RuntimeModelCatalog = {
-    defaultSession: model.id,
-    defaultMetadataGeneration: null,
-    embedding: null,
-    sessionModels: () => [model],
-    metadataModels: () => [],
-    hasSession: (id) => id === model.id,
-  };
-  let spawnEnv: NodeJS.ProcessEnv | undefined;
-  const smoke = createPiProviderSmoke(loadConfig({ THT_SECRETS_FILE: secret }), {
-    modelCatalog,
-    authProviders: () => new Set(),
-    readModelsStore: () => undefined,
-    spawnFn: (_command, _args, options) => {
-      spawnEnv = options.env;
-      return successfulProviderChild();
-    },
-  });
-  try {
-    await expect(smoke({
-      provider: "openai", model: "test-model", reasoning: "medium", timeoutMs: 750,
-    })).resolves.toBeUndefined();
-    expect(spawnEnv?.ZAI_API_KEY).toBe("catalog-secret");
-    expect(spawnEnv).not.toHaveProperty("OPENAI_API_KEY");
-    expect(spawnEnv).not.toHaveProperty("THT_MODEL_API_KEY");
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
 // Catches an isolated smoke agent that copies auth.json but drops the selected custom
 // provider/model from models.json, causing set_model to fail before the real request.
 test("provider smoke reaches the selected custom provider from an isolated models.json", async () => {
@@ -300,22 +254,11 @@ test("provider smoke makes one configured request from an isolated no-capability
       });
     }
   });
-  const smokeModel: RuntimeModel = {
-    id: "zai/catalog-glm", provider: "zai", model: "catalog-glm", label: "GLM",
-    upstreamModel: "glm-5.2", authentication: { mode: "pi_auth" },
-    sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true },
-  };
-  const smokeCatalog: RuntimeModelCatalog = {
-    defaultSession: smokeModel.id, defaultMetadataGeneration: null, embedding: null,
-    sessionModels: () => [smokeModel], metadataModels: () => [],
-    hasSession: (id) => id === smokeModel.id,
-  };
   const smoke = createPiProviderSmoke(loadConfig({
     THT_HARNESS_DIR: "/app/harness",
     PI_BIN: "/usr/local/bin/pi",
     THT_DATA_ROOT: "/mounted-workflow-state",
   }), {
-    modelCatalog: smokeCatalog,
     spawnFn: (...args) => {
       spawns.push(args);
       expect(args[2].cwd).not.toBe("/app/harness");
@@ -335,7 +278,7 @@ test("provider smoke makes one configured request from an isolated no-capability
   });
 
   await expect(smoke({
-    provider: "zai", model: "catalog-glm", reasoning: "medium", timeoutMs: 750,
+    provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750,
   })).resolves.toBeUndefined();
   expect(spawns).toHaveLength(1);
   expect(spawns[0][0]).toBe("/usr/local/bin/pi");