diff --git a/.gitignore b/.gitignore index 803d0cd0..2ba0009e 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ config/ca-chain.pem # ThothII deployment configuration and secret values (keep only the README tracked) deploy/.env +deploy/env/local.env deploy/compose.connector-secrets.local.yaml deploy/compose.psd-local.yaml deploy/workspaces/psd.yaml diff --git a/CONTEXT.md b/CONTEXT.md index dc9940f9..3c538a8d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -267,6 +267,21 @@ ridefinirne provider, endpoint o capacità. dell'Installation Model Catalog richiesta da uno specifico runtime. Può essere rigenerata integralmente dalla configurazione dell'installazione. +## Distribuzione del prodotto + +**Customer-Hosted Installation** — Un'installazione eseguita interamente nel trust boundary +controllato dall'organizzazione cliente, inclusi eventuali tenant cloud privati. Credenziali, +domande, prompt, metadati e risultati non attraversano quel boundary. +_Avoid_: on-premise deployment, self-managed deployment + +**Community Edition** — La distribuzione open source utilizzabile gratuitamente anche in +produzione e capace di eseguire il workflow fondamentale completo. +_Avoid_: free tier, trial edition + +**Enterprise Edition** — La distribuzione con licenza commerciale che aggiunge governance +organizzativa, esercizio production-grade e industrializzazione alla Community Edition. +_Avoid_: paid tier, pro edition + ## Catalogo dei metadati **Workspace Database** — Il database che appartiene a un solo workspace e non può essere @@ -420,6 +435,12 @@ diventa Catalog Metadata. **Sensitive Data Flag** — La classificazione binaria umana applicata a una Catalog Column. Può essere impostata liberamente dall'amministratore anche in contrasto con una valutazione automatica. +**Sensitivity Reason** — La motivazione sanificata persistita insieme al Sensitive Data Flag +quando l'amministratore salva una Sensitivity Review Draft. È Catalog Metadata della colonna, non +history della run; viene rimossa quando il flag torna non-sensitive e può essere assente per una +classificazione manuale priva di valutazione locale. +_Avoid_: AI reasoning, source evidence + **Local Sensitivity Assessment** — La valutazione locale, non autoritativa e priva di LLM di una Catalog Column, basata su metadati e contenuto sorgente, con esito `sensitive`, `non_sensitive` oppure `unknown`. @@ -451,12 +472,13 @@ _Avoid_: Sensitive Data Suggestion Run, AI analysis **Sensitivity Review Draft** — La proposta transitoria che associa alle colonne selezionate una Local Sensitivity Assessment e le relative evidenze sanificate. Non modifica il Sensitive Data Flag -finché l'amministratore non salva le proprie decisioni e viene scartata al reload. +né la Sensitivity Reason finché l'amministratore non salva le proprie decisioni e viene scartata al +reload. _Avoid_: automatic flag **Sensitivity Analysis Event** — Una riga testuale ordinata e sanificata che registra l'avvio, -l'esito o l'errore di una Sensitivity Analysis Run senza conservare contenuti sorgente, output grezzi -del detector o proposte per colonna. +l'avanzamento per fase e batch, l'esito o l'errore di una Sensitivity Analysis Run senza conservare +contenuti sorgente, output grezzi del detector o proposte per colonna. _Avoid_: Sensitive Data Suggestion Event **Introspection Capability** — Una categoria di struttura fisica che una Database Binding diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 5560f398..70498ec2 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -98,7 +98,7 @@ column. The KPI strip reads installation-wide or selected-database aggregates fr description history, and sensitive-field review/history use the production APIs in right-side drawers rather than prototype fixtures; closing a history drawer does not stop its background run. -Sensitive-field review is now driven by the versioned local `sensitivity-v2` policy, not by a +Sensitive-field review is now driven by the versioned local `sensitivity-v4` policy, not by a catalog model. The backend reads selected source tables through read-only, database-specific adapters and makes every `sensitive | non_sensitive` draft decision in the TypeScript `SensitivityClassifier`. A single validated match protects the column. Tables up to 1,000 rows are @@ -111,6 +111,9 @@ evidence is CPU-only, offline, opt-in, and never replaces the deterministic deci default; see `docs/reports/2026-09-02-psd-sensitivity-shadow.md`. The v2 comparison completed all 2,275 columns: CPU NER added 18 sensitive proposals and increased warm runtime from 50.1 to 61.3 seconds; see `docs/reports/2026-09-03-psd-progressive-sensitivity-shadow.md`. +Version 4 excludes declared `bigint` primary-key columns and conventionally named `pk bigint` +columns before source inspection, reporting both as non-informative structural identifiers while +distinguishing declared constraints from inferred roles. Physical membership, source comments, column types/default/nullability/PK positions, and constraint-level ordered FK pairs are @@ -176,8 +179,12 @@ no generative model decides the result. Its `sensitive` or `non_sensitive` asses unsaved draft until the human reviews and saves any chosen flag changes, including a downgrade to non-sensitive. Coverage is reported separately; interrupted history may count unprocessed columns. Each started analysis records a separate Sensitivity Analysis Run with aggregate counters and safe -ordered events. This operational history never stores per-column assessments, source values, -matched spans, prompts, or free-form diagnostics; reloading still discards an unsaved review draft. +ordered events. The progress drawer opens before the synchronous request completes, polls the run, +and displays sanitized source-scan and local-NER phase/batch activity while classification is in +progress. This operational history never stores per-column assessments, source values, +matched spans, prompts, or free-form diagnostics. Saving a sensitive decision persists a sanitized +Sensitivity Reason as column Catalog Metadata alongside the human-owned flag; clearing the flag +clears that reason. Reloading still discards an unsaved review draft. For unprotected columns, up to five source rows and five representative non-null values may be sent transiently to the configured model provider. Protected columns are omitted from source reads and replaced in the prompt by deterministic plausible values derived only from their metadata. Existing diff --git a/backend/src/app.ts b/backend/src/app.ts index d9d2765c..9cc268d2 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -33,7 +33,11 @@ import { ReadinessManager } from "./runtime/readiness-manager.js"; import { MaintenanceBarrier } from "./runtime/maintenance-gate.js"; import { WorkspaceRegistry } from "./workspaces/registry.js"; import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js"; -import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js"; +import { + workspaceRoutes, + type WorkspaceDatabaseTester, + type WorkspaceDiagnoser, +} from "./routes/workspaces.js"; import { piManagementRoutes } from "./routes/pi-management.js"; import { supportsSessionRuntime } from "./workspaces/bindings.js"; import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js"; @@ -83,6 +87,7 @@ export interface BuildAppDeps { hub?: SseHub; workspaceRegistry?: WorkspaceRegistry; workspaceDiagnoser?: WorkspaceDiagnoser; + workspaceDatabaseTester?: WorkspaceDatabaseTester; workspaceSecretStore?: WorkspaceSecretStore; catalogRepository?: CatalogRepository; catalogService?: CatalogService; @@ -230,6 +235,10 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc catalogPostgresAccess, catalogOperationCoordinator, ); + const workspaceDatabaseTester = deps?.workspaceDatabaseTester ?? (async (workspaceId: string) => { + const database = await catalogRepository.getByWorkspace(workspaceId); + return database ? catalogService.test(database) : undefined; + }); const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository); const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService ?? new CatalogLogicalRelationshipService(catalogRepository); @@ -485,6 +494,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc diagnose: workspaceDiagnoser, authDiagnoser, secretStore: workspaceSecretStore, + testDatabaseConnection: workspaceDatabaseTester, }); catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator }); catalogTableRoutes(app, { diff --git a/backend/src/catalog/memory-repository.ts b/backend/src/catalog/memory-repository.ts index 81a17085..d0fe18be 100644 --- a/backend/src/catalog/memory-repository.ts +++ b/backend/src/catalog/memory-repository.ts @@ -83,8 +83,10 @@ export class MemoryCatalogRepository implements CatalogRepository { const tableIds = new Set(tables.map((table) => table.id)); const columns = [...this.columns.values()] .filter((column) => tableIds.has(column.tableId)); - const relationships = [...this.relationships.values()] - .filter((relationship) => selectedDatabaseIds.has(relationship.databaseId)); + const relationships = [ + ...this.relationships.values(), + ...this.logicalRelationships.values(), + ].filter((relationship) => selectedDatabaseIds.has(relationship.databaseId)); return createCatalogMetrics(databaseId, { tables: tables.length, @@ -255,6 +257,7 @@ export class MemoryCatalogRepository implements CatalogRepository { description: string | null, generatedDescription: string | null, sensitive?: boolean, + sensitivityReason?: string | null, ): Promise { const current = await this.getColumn(databaseId, tableId, columnId); if (!current || current.version !== expectedVersion) return undefined; @@ -263,6 +266,9 @@ export class MemoryCatalogRepository implements CatalogRepository { description, generatedDescription, sensitive: sensitive ?? current.sensitive, + sensitivityReason: sensitive === false + ? null + : sensitivityReason === undefined ? current.sensitivityReason : sensitivityReason, version: current.version + 1, updatedAt: new Date().toISOString(), }; @@ -276,10 +282,26 @@ export class MemoryCatalogRepository implements CatalogRepository { targetIds: readonly string[], ): Promise { const selectedTargetIds = [...new Set(targetIds)]; - if (!this.records.has(databaseId) || selectedTargetIds.length === 0) { + if (!this.records.has(databaseId)) { return undefined; } const now = new Date().toISOString(); + if (target === "database_columns") { + const targets = [...this.columns.values()].filter((column) => ( + this.tables.get(column.tableId)?.databaseId === databaseId + )); + const copied = targets.filter((column) => Boolean(column.generatedDescription?.trim())); + for (const column of copied) { + this.columns.set(column.id, { + ...column, + description: column.generatedDescription, + version: column.version + 1, + updatedAt: now, + }); + } + return { copied: copied.length, skipped: targets.length - copied.length }; + } + if (selectedTargetIds.length === 0) return undefined; if (target === "tables") { const targets = selectedTargetIds.map((id) => this.tables.get(id)); if (targets.some((table) => !table || table.databaseId !== databaseId)) return undefined; @@ -687,18 +709,22 @@ export class MemoryCatalogRepository implements CatalogRepository { const tables = [...this.tables.values()].filter((table) => selected.has(table.databaseId)); const tableIds = new Set(tables.map((table) => table.id)); const columns = [...this.columns.values()].filter((column) => tableIds.has(column.tableId)); - const relationships = [...this.relationships.values()] + const physicalRelationships = [...this.relationships.values()] .filter((relationship) => selected.has(relationship.databaseId)); + const logicalRelationships = [...this.logicalRelationships.values()] + .filter((relationship) => selected.has(relationship.databaseId)); + const relationshipCount = physicalRelationships.length + logicalRelationships.length; if (target === "tables") { for (const table of tables) this.deleteTable(table.id); this.markCatalogIncomplete(selectedDatabaseIds); - return { tables: tables.length, columns: columns.length, relationships: relationships.length }; + return { tables: tables.length, columns: columns.length, relationships: relationshipCount }; } - for (const relationship of relationships) this.relationships.delete(relationship.id); + for (const relationship of physicalRelationships) this.relationships.delete(relationship.id); + for (const relationship of logicalRelationships) this.logicalRelationships.delete(relationship.id); for (const databaseId of selectedDatabaseIds) this.refreshForeignKeyFlags(databaseId); this.markCatalogIncomplete(selectedDatabaseIds); - return { tables: 0, columns: 0, relationships: relationships.length }; + return { tables: 0, columns: 0, relationships: relationshipCount }; } async deleteTableMetadata( @@ -736,14 +762,23 @@ export class MemoryCatalogRepository implements CatalogRepository { return { tables: 0, columns: columns.length, relationships: 0 }; } - const relationships = [...this.relationships.values()].filter((relationship) => ( + const physicalRelationships = [...this.relationships.values()].filter((relationship) => ( relationship.databaseId === databaseId && (selected.has(relationship.sourceTableId) || selected.has(relationship.targetTableId)) )); - for (const relationship of relationships) this.relationships.delete(relationship.id); + const logicalRelationships = [...this.logicalRelationships.values()].filter((relationship) => ( + relationship.databaseId === databaseId + && (selected.has(relationship.sourceTableId) || selected.has(relationship.targetTableId)) + )); + for (const relationship of physicalRelationships) this.relationships.delete(relationship.id); + for (const relationship of logicalRelationships) this.logicalRelationships.delete(relationship.id); this.refreshForeignKeyFlags(databaseId); this.markCatalogIncomplete([databaseId]); - return { tables: 0, columns: 0, relationships: relationships.length }; + return { + tables: 0, + columns: 0, + relationships: physicalRelationships.length + logicalRelationships.length, + }; } async planSchemaSync( @@ -927,6 +962,7 @@ export class MemoryCatalogRepository implements CatalogRepository { description: null, generatedDescription: null, sensitive: false, + sensitivityReason: null, lastSyncedDatabaseVersion: expectedDatabaseVersion, lastSyncedAt: now, version: 1, diff --git a/backend/src/catalog/migrate.ts b/backend/src/catalog/migrate.ts index 55b50c14..c5af8d39 100644 --- a/backend/src/catalog/migrate.ts +++ b/backend/src/catalog/migrate.ts @@ -14,6 +14,7 @@ import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_ import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js"; import * as canonicalModelIdsMigration from "./migrations/010_canonical_model_ids.js"; import * as localSensitivityAnalysisMigration from "./migrations/011_local_sensitivity_analysis.js"; +import * as sensitivityReasonMigration from "./migrations/012_sensitivity_reason.js"; const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL; const host = process.env.THT_CATALOG_DB_HOST; @@ -50,6 +51,7 @@ const provider: MigrationProvider = { "009_ai_token_usage": aiTokenUsageMigration, "010_canonical_model_ids": canonicalModelIdsMigration, "011_local_sensitivity_analysis": localSensitivityAnalysisMigration, + "012_sensitivity_reason": sensitivityReasonMigration, }; }, }; diff --git a/backend/src/catalog/migrations/012_sensitivity_reason.ts b/backend/src/catalog/migrations/012_sensitivity_reason.ts new file mode 100644 index 00000000..19a0d94b --- /dev/null +++ b/backend/src/catalog/migrations/012_sensitivity_reason.ts @@ -0,0 +1,12 @@ +import type { Kysely } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable("catalog_columns") + .addColumn("sensitivity_reason", "text") + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable("catalog_columns").dropColumn("sensitivity_reason").execute(); +} diff --git a/backend/src/catalog/repository.ts b/backend/src/catalog/repository.ts index c591af45..331c773e 100644 --- a/backend/src/catalog/repository.ts +++ b/backend/src/catalog/repository.ts @@ -118,6 +118,7 @@ interface CatalogColumnTable { description: string | null; generatedDescription: string | null; sensitive: Generated; + sensitivityReason: Generated; lastSyncedDatabaseVersion: number | null; lastSyncedAt: Timestamp | null; version: Generated; @@ -360,6 +361,7 @@ function serializeColumn(row: Selectable, foreignKeyCount = description: row.description, generatedDescription: row.generatedDescription, sensitive: row.sensitive, + sensitivityReason: row.sensitivityReason, lastSyncedDatabaseVersion: row.lastSyncedDatabaseVersion, lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(), version: row.version, @@ -516,10 +518,18 @@ export class KyselyCatalogRepository implements CatalogRepository { relationship_metrics AS ( SELECT count(*)::int AS relationships, - max(catalog_relationships.updated_at) AS updated_at - FROM catalog_relationships - INNER JOIN selected_databases - ON selected_databases.id = catalog_relationships.database_id + max(relationship.updated_at) AS updated_at + FROM ( + SELECT catalog_relationships.updated_at + FROM catalog_relationships + INNER JOIN selected_databases + ON selected_databases.id = catalog_relationships.database_id + UNION ALL + SELECT catalog_logical_relationships.updated_at + FROM catalog_logical_relationships + INNER JOIN selected_databases + ON selected_databases.id = catalog_logical_relationships.database_id + ) AS relationship ) SELECT (SELECT count(*)::int FROM selected_databases) AS "databaseCount", @@ -728,6 +738,7 @@ export class KyselyCatalogRepository implements CatalogRepository { description: string | null, generatedDescription: string | null, sensitive?: boolean, + sensitivityReason?: string | null, ): Promise { const belongs = await this.db.selectFrom("catalogTables").select("id") .where("id", "=", tableId).where("databaseId", "=", databaseId).executeTakeFirst(); @@ -736,6 +747,9 @@ export class KyselyCatalogRepository implements CatalogRepository { description, generatedDescription, ...(sensitive === undefined ? {} : { sensitive }), + ...(sensitive === false + ? { sensitivityReason: null } + : sensitivityReason === undefined ? {} : { sensitivityReason }), version: sql`version + 1`, updatedAt: sql`now()`, }).where("id", "=", columnId).where("tableId", "=", tableId) @@ -749,7 +763,7 @@ export class KyselyCatalogRepository implements CatalogRepository { targetIds: readonly string[], ): Promise { const selectedTargetIds = [...new Set(targetIds)]; - if (selectedTargetIds.length === 0) return undefined; + if (target !== "database_columns" && selectedTargetIds.length === 0) return undefined; return await this.db.transaction().execute(async (trx) => { const database = await trx.selectFrom("workspaceDatabases").select("id") .where("id", "=", databaseId).forUpdate().executeTakeFirst(); @@ -784,24 +798,30 @@ export class KyselyCatalogRepository implements CatalogRepository { const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns") .select(["id", "generatedDescription"]) .where("tableId", "in", tableRows.map((table) => table.id)) - .where("id", "in", selectedTargetIds) + .$if(target === "columns", (query) => query.where("id", "in", selectedTargetIds)) .orderBy("id") .forUpdate() .execute(); - if (rows.length !== selectedTargetIds.length) return undefined; + if (target === "columns" && rows.length !== selectedTargetIds.length) return undefined; const copiedIds = rows .filter((row) => Boolean(row.generatedDescription?.trim())) .map((row) => row.id); if (copiedIds.length > 0) { - await trx.updateTable("catalogColumns").set({ + let update = trx.updateTable("catalogColumns").set({ description: sql`generated_description`, version: sql`version + 1`, updatedAt: sql`now()`, - }).where("id", "in", copiedIds).execute(); + }); + update = target === "database_columns" + ? update + .where("tableId", "in", tableRows.map((table) => table.id)) + .where(sql`nullif(btrim(generated_description), '') is not null`) + : update.where("id", "in", copiedIds); + await update.execute(); } return { copied: copiedIds.length, - skipped: selectedTargetIds.length - copiedIds.length, + skipped: rows.length - copiedIds.length, }; }); } @@ -1261,16 +1281,25 @@ export class KyselyCatalogRepository implements CatalogRepository { if (databases.length !== selectedDatabaseIds.length) return undefined; if (target === "relationships") { - const count = await trx.selectFrom("catalogRelationships") + const physicalCount = await trx.selectFrom("catalogRelationships") .select(sql`count(*)::int`.as("count")) .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst(); + const logicalCount = await trx.selectFrom("catalogLogicalRelationships") + .select(sql`count(*)::int`.as("count")) + .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst(); + await trx.deleteFrom("catalogLogicalRelationships") + .where("databaseId", "in", selectedDatabaseIds).execute(); await trx.deleteFrom("catalogRelationships") .where("databaseId", "in", selectedDatabaseIds).execute(); await trx.updateTable("workspaceDatabases").set({ schemaSyncedVersion: null, schemaSyncedAt: null, }).where("id", "in", selectedDatabaseIds).execute(); - return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) }; + return { + tables: 0, + columns: 0, + relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0), + }; } const tableCount = await trx.selectFrom("catalogTables") @@ -1280,7 +1309,10 @@ export class KyselyCatalogRepository implements CatalogRepository { .innerJoin("catalogTables", "catalogTables.id", "catalogColumns.tableId") .select(sql`count(*)::int`.as("count")) .where("catalogTables.databaseId", "in", selectedDatabaseIds).executeTakeFirst(); - const relationshipCount = await trx.selectFrom("catalogRelationships") + const physicalRelationshipCount = await trx.selectFrom("catalogRelationships") + .select(sql`count(*)::int`.as("count")) + .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst(); + const logicalRelationshipCount = await trx.selectFrom("catalogLogicalRelationships") .select(sql`count(*)::int`.as("count")) .where("databaseId", "in", selectedDatabaseIds).executeTakeFirst(); await trx.deleteFrom("catalogTables") @@ -1292,7 +1324,8 @@ export class KyselyCatalogRepository implements CatalogRepository { return { tables: Number(tableCount?.count ?? 0), columns: Number(columnCount?.count ?? 0), - relationships: Number(relationshipCount?.count ?? 0), + relationships: Number(physicalRelationshipCount?.count ?? 0) + + Number(logicalRelationshipCount?.count ?? 0), }; }); } @@ -1326,13 +1359,34 @@ export class KyselyCatalogRepository implements CatalogRepository { return { tables: 0, columns: Number(count?.count ?? 0), relationships: 0 }; } - const count = await trx.selectFrom("catalogRelationships") + const physicalCount = await trx.selectFrom("catalogRelationships") .select(sql`count(*)::int`.as("count")) .where("databaseId", "=", databaseId) .where((eb) => eb.or([ eb("sourceTableId", "in", selectedTableIds), eb("targetTableId", "in", selectedTableIds), ])).executeTakeFirst(); + const selectedColumnIds = (await trx.selectFrom("catalogColumns") + .select("id") + .where("tableId", "in", selectedTableIds) + .execute()).map((column) => column.id); + const logicalCount = selectedColumnIds.length === 0 + ? undefined + : await trx.selectFrom("catalogLogicalRelationships") + .select(sql`count(*)::int`.as("count")) + .where("databaseId", "=", databaseId) + .where((eb) => eb.or([ + eb("sourceColumnId", "in", selectedColumnIds), + eb("targetColumnId", "in", selectedColumnIds), + ])).executeTakeFirst(); + if (selectedColumnIds.length > 0) { + await trx.deleteFrom("catalogLogicalRelationships") + .where("databaseId", "=", databaseId) + .where((eb) => eb.or([ + eb("sourceColumnId", "in", selectedColumnIds), + eb("targetColumnId", "in", selectedColumnIds), + ])).execute(); + } await trx.deleteFrom("catalogRelationships") .where("databaseId", "=", databaseId) .where((eb) => eb.or([ @@ -1343,7 +1397,11 @@ export class KyselyCatalogRepository implements CatalogRepository { schemaSyncedVersion: null, schemaSyncedAt: null, }).where("id", "=", databaseId).execute(); - return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) }; + return { + tables: 0, + columns: 0, + relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0), + }; }); } diff --git a/backend/src/catalog/sensitivity-analysis-runner.ts b/backend/src/catalog/sensitivity-analysis-runner.ts index 23f10af9..971726af 100644 --- a/backend/src/catalog/sensitivity-analysis-runner.ts +++ b/backend/src/catalog/sensitivity-analysis-runner.ts @@ -116,6 +116,15 @@ export class SensitivityAnalysisRunner { ); ensureActive(signal); }, + async (message) => { + ensureActive(signal); + await this.repository.appendSensitivityAnalysisEvent( + started.id, + "info", + message, + ); + ensureActive(signal); + }, ); ensureActive(signal); const suggestedSensitive = suggestions.filter( diff --git a/backend/src/catalog/sensitivity-analysis-service.ts b/backend/src/catalog/sensitivity-analysis-service.ts index 95c779d1..1395146c 100644 --- a/backend/src/catalog/sensitivity-analysis-service.ts +++ b/backend/src/catalog/sensitivity-analysis-service.ts @@ -12,7 +12,7 @@ import type { } from "./types.js"; export type { SensitivityAnalysisScope } from "./types.js"; -export const SENSITIVITY_POLICY_VERSION = "sensitivity-v2"; +export const SENSITIVITY_POLICY_VERSION = "sensitivity-v4"; interface SelectedColumn { table: CatalogTable; @@ -116,6 +116,7 @@ export class SensitivityAnalysisService { signal: AbortSignal, onPrepared?: (total: number) => void | Promise, onProgress?: (processed: number, suggestions: readonly SensitivityReviewItem[]) => void | Promise, + onActivity?: (message: string) => void | Promise, ): Promise { const configuredNerBudget = this.options.nerBudgetMs ?? 10_000; const nerBudget: SensitivityNerBudget = { @@ -144,7 +145,12 @@ export class SensitivityAnalysisService { columns: items.map(({ column }) => column), }; }); - const assessments = await this.classifier.assess(tableTargets, signal, nerBudget); + const assessments = await this.classifier.assess( + tableTargets, + signal, + nerBudget, + onActivity, + ); ensureActive(signal); const assessmentById = new Map(assessments.map((assessment) => [ assessment.columnId, diff --git a/backend/src/catalog/sensitivity-classifier.ts b/backend/src/catalog/sensitivity-classifier.ts index 70383680..b4586b45 100644 --- a/backend/src/catalog/sensitivity-classifier.ts +++ b/backend/src/catalog/sensitivity-classifier.ts @@ -128,6 +128,25 @@ function metadataEvidence(column: CatalogColumn): SensitivityEvidence | undefine return ruleId ? { kind: "metadata", ruleId } : undefined; } +function nonSensitiveStructuralEvidence(column: CatalogColumn): SensitivityEvidence | undefined { + if (column.dataType.trim().toLowerCase() !== "bigint") return undefined; + if (column.isPrimaryKey || column.primaryKeyPosition !== null) { + return { + kind: "type", + ruleId: "type.bigint_primary_key_non_informative", + label: "non-informative bigint primary key", + }; + } + if (normalizedName(column.name) === "pk") { + return { + kind: "metadata", + ruleId: "metadata.bigint_pk_identifier_non_informative", + label: "non-informative conventional bigint primary-key identifier", + }; + } + return undefined; +} + function sensitiveNameRule(value: string): string | undefined { const name = normalizedName(value); if (DIRECT_IDENTIFIER_NAMES.has(name)) { @@ -299,6 +318,7 @@ function contentEvidence(value: string): SensitivityEvidence | undefined { interface ColumnState { column: CatalogColumn; evidence: SensitivityEvidence[]; + nonSensitiveEvidence?: SensitivityEvidence; observedValues: number; nerCandidates: string[]; coverage: "metadata" | "complete" | "sampled" | "no_values"; @@ -323,14 +343,16 @@ export class SensitivityClassifier { targets: readonly SensitivityTableTarget[], signal: AbortSignal, sharedNerBudget?: SensitivityNerBudget, + onActivity?: (message: string) => void | Promise, ): Promise { const now = this.options.now ?? Date.now; const maxNerValuesPerColumn = boundedCount(this.options.maxNerValuesPerColumn, 8, 8); const states = new Map(); for (const target of targets) { for (const column of target.columns) { - const metadataMatch = metadataEvidence(column); - const binary = UNSUPPORTED_BINARY_TYPE.test(column.dataType); + const nonSensitiveEvidence = nonSensitiveStructuralEvidence(column); + const metadataMatch = nonSensitiveEvidence ? undefined : metadataEvidence(column); + const binary = !nonSensitiveEvidence && UNSUPPORTED_BINARY_TYPE.test(column.dataType); states.set(column.id, { column, evidence: metadataMatch @@ -338,9 +360,10 @@ export class SensitivityClassifier { : binary ? [{ kind: "type", ruleId: "type.binary_uninspectable" }] : [], + ...(nonSensitiveEvidence ? { nonSensitiveEvidence } : {}), observedValues: 0, nerCandidates: [], - coverage: metadataMatch || binary ? "metadata" : "no_values", + coverage: nonSensitiveEvidence || metadataMatch || binary ? "metadata" : "no_values", sampledTarget: 0, }); } @@ -349,6 +372,12 @@ export class SensitivityClassifier { const completeTables = new Set(); for (const [phaseIndex, phase] of SENSITIVITY_SAMPLE_PHASES.entries()) { for (let offset = 0; offset < targets.length; offset += MAX_CONCURRENT_TABLE_SCANS) { + signal.throwIfAborted(); + const batchNumber = Math.floor(offset / MAX_CONCURRENT_TABLE_SCANS) + 1; + const batchCount = Math.ceil(targets.length / MAX_CONCURRENT_TABLE_SCANS); + await onActivity?.( + `Scanning source data: pass ${phaseIndex + 1} of ${SENSITIVITY_SAMPLE_PHASES.length}, table batch ${batchNumber} of ${batchCount}.`, + ); signal.throwIfAborted(); const peerController = new AbortController(); const scanSignal = AbortSignal.any([signal, peerController.signal]); @@ -357,7 +386,7 @@ export class SensitivityClassifier { if (completeTables.has(target.table.id)) return; const columns = target.columns.filter((column) => { const state = states.get(column.id)!; - return state.evidence.length === 0 + return state.evidence.length === 0 && !state.nonSensitiveEvidence && (!phase.deepTextOnly || DEEP_TEXT_TYPE.test(column.dataType)); }); if (columns.length === 0) return; @@ -411,14 +440,14 @@ export class SensitivityClassifier { && nerBudget.remainingMs > 0) { const maxCandidates = boundedCount(this.options.maxNerCandidatesPerTable, 2, 1_024); const threshold = this.options.nerConfidenceThreshold ?? 0.8; - for (const target of targets) { + for (const [targetIndex, target] of targets.entries()) { signal.throwIfAborted(); if (nerBudget.remainingMs <= 0) break; const candidates: LocalNerCandidate[] = []; candidateSelection: for (let valueIndex = 0; valueIndex < maxNerValuesPerColumn; valueIndex += 1) { for (const column of target.columns) { const state = states.get(column.id)!; - if (state.evidence.length > 0) continue; + if (state.evidence.length > 0 || state.nonSensitiveEvidence) continue; const text = state.nerCandidates[valueIndex]; if (text === undefined) continue; candidates.push({ columnId: column.id, text }); @@ -426,6 +455,10 @@ export class SensitivityClassifier { } } if (candidates.length === 0) continue; + await onActivity?.( + `Running local entity detection: table ${targetIndex + 1} of ${targets.length}.`, + ); + signal.throwIfAborted(); const startedAt = now(); const deadline = startedAt + nerBudget.remainingMs; try { @@ -464,17 +497,21 @@ export class SensitivityClassifier { return targets.flatMap((target) => target.columns.map((column) => { const state = states.get(column.id)!; const sensitive = state.evidence.length > 0; - const coverage = state.observedValues === 0 && !sensitive ? "no_values" : state.coverage; + const coverage = state.nonSensitiveEvidence + ? "metadata" + : state.observedValues === 0 && !sensitive ? "no_values" : state.coverage; const coverageEvidence: SensitivityEvidence[] = sensitive ? state.evidence - : [{ - kind: "coverage", - ruleId: coverage === "complete" - ? "coverage.complete" - : coverage === "no_values" - ? "coverage.no_values" - : `coverage.sampled_${state.sampledTarget}`, - }]; + : state.nonSensitiveEvidence + ? [state.nonSensitiveEvidence] + : [{ + kind: "coverage", + ruleId: coverage === "complete" + ? "coverage.complete" + : coverage === "no_values" + ? "coverage.no_values" + : `coverage.sampled_${state.sampledTarget}`, + }]; return { columnId: column.id, assessment: sensitive ? "sensitive" : "non_sensitive", diff --git a/backend/src/catalog/sync-worker.ts b/backend/src/catalog/sync-worker.ts index 8f837b63..a880306b 100644 --- a/backend/src/catalog/sync-worker.ts +++ b/backend/src/catalog/sync-worker.ts @@ -39,7 +39,10 @@ function safeFailure(error: unknown): { code: string; message: string } { }; } if (error instanceof CatalogConnectorError) { - return { code: "schema_introspection_failed", message: "The database schema could not be read safely." }; + return { + code: "schema_introspection_failed", + message: "The database schema could not be read. Check the connection and credentials, then try again.", + }; } return { code: "schema_sync_failed", message: "Schema synchronization failed." }; } @@ -64,7 +67,6 @@ export class CatalogSyncWorker { } async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise { - this.assertReady(database); const uniqueTableIds = [...new Set(tableIds)]; if (scope === "columns") { const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId))); @@ -177,7 +179,6 @@ export class CatalogSyncWorker { if (!database || database.version !== claimed.requestedDatabaseVersion) { throw new CatalogConflictError("Database binding changed before synchronization started"); } - this.assertReady(database); const progress: CatalogSchemaScanProgress = async (phase, counts) => { await this.checkCancelled(runId); await this.repository.updateSyncRun(runId, { @@ -277,12 +278,6 @@ export class CatalogSyncWorker { } } - private assertReady(database: WorkspaceDatabase): void { - if (database.connectionStatus !== "reachable" || database.testedVersion !== database.version) { - throw new CatalogConflictError("Test the current database binding before synchronizing its schema"); - } - } - private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void { const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const; for (const name of required) { diff --git a/backend/src/catalog/types.ts b/backend/src/catalog/types.ts index a177cef0..d9ef63e8 100644 --- a/backend/src/catalog/types.ts +++ b/backend/src/catalog/types.ts @@ -102,6 +102,7 @@ export interface CatalogColumn { description: string | null; generatedDescription: string | null; sensitive: boolean; + sensitivityReason: string | null; lastSyncedDatabaseVersion: number | null; lastSyncedAt: string | null; version: number; @@ -195,7 +196,7 @@ export interface CatalogLogicalRelationshipCandidate { export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships"; export type CatalogTableMetadataDeleteTarget = "columns" | "relationships"; -export type CatalogDescriptionTarget = "tables" | "columns"; +export type CatalogDescriptionTarget = "tables" | "columns" | "database_columns"; export interface CatalogMetadataDeleteCounts { tables: number; @@ -463,6 +464,7 @@ export interface CatalogRepository { description: string | null, generatedDescription: string | null, sensitive?: boolean, + sensitivityReason?: string | null, ): Promise; consolidateGeneratedDescriptions( databaseId: string, diff --git a/backend/src/routes/catalog-description-consolidation.ts b/backend/src/routes/catalog-description-consolidation.ts index fd9ad806..973c7d6a 100644 --- a/backend/src/routes/catalog-description-consolidation.ts +++ b/backend/src/routes/catalog-description-consolidation.ts @@ -9,10 +9,13 @@ import { } from "../catalog/types.js"; const idSchema = z.uuid(); -const consolidationSchema = z.object({ - target: z.enum(["tables", "columns"]), - targetIds: z.array(idSchema).min(1).max(10_000), -}).strict(); +const consolidationSchema = z.discriminatedUnion("target", [ + z.object({ + target: z.enum(["tables", "columns"]), + targetIds: z.array(idSchema).min(1).max(10_000), + }).strict(), + z.object({ target: z.literal("database_columns") }).strict(), +]); function manage(request: FastifyRequest, reply: FastifyReply) { return isPrincipalContext(requirePermission(request, reply, "database.manage")); @@ -49,7 +52,7 @@ export function catalogDescriptionConsolidationRoutes( try { const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); const input = consolidationSchema.parse(request.body); - const targetIds = [...new Set(input.targetIds)]; + const targetIds = "targetIds" in input ? [...new Set(input.targetIds)] : []; const result = await deps.operations.run( databaseId, async () => await deps.repository.consolidateGeneratedDescriptions( diff --git a/backend/src/routes/catalog-schema.ts b/backend/src/routes/catalog-schema.ts index d5182b51..979a5244 100644 --- a/backend/src/routes/catalog-schema.ts +++ b/backend/src/routes/catalog-schema.ts @@ -19,8 +19,14 @@ const metadataSchema = z.object({ description: z.string().max(20_000).nullable().optional(), generatedDescription: z.string().max(20_000).nullable().optional(), sensitive: z.boolean().optional(), + sensitivityReason: z.string().max(2_000).nullable().optional(), }).strict().refine((value) => ( - "description" in value || "generatedDescription" in value || "sensitive" in value + "description" in value + || "generatedDescription" in value + || "sensitive" in value + || "sensitivityReason" in value +)).refine((value) => ( + value.sensitivityReason == null || value.sensitive === true )); const createRunSchema = z.object({ version: z.number().int().positive(), @@ -56,7 +62,10 @@ function safeError(reply: FastifyReply, error: unknown) { return reply.code(409).send({ code: "schema_sync_conflict", message: error.message }); } if (error instanceof CatalogConnectorError) { - return reply.code(502).send({ code: "schema_introspection_failed", message: "The database schema could not be read safely." }); + return reply.code(502).send({ + code: "schema_introspection_failed", + message: "The database schema could not be read. Check the connection and credentials, then try again.", + }); } if (error instanceof z.ZodError) { return reply.code(400).send({ code: "schema_request_invalid", message: "Schema request is invalid." }); @@ -113,6 +122,12 @@ export function catalogSchemaRoutes( if (current.version !== input.version) { return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." }); } + const nextSensitive = input.sensitive ?? current.sensitive; + const nextSensitivityReason = nextSensitive + ? ("sensitivityReason" in input + ? normalized(input.sensitivityReason ?? null) + : current.sensitivityReason) + : null; const updated = await deps.repository.updateColumnMetadata( databaseId, tableId, @@ -123,6 +138,7 @@ export function catalogSchemaRoutes( ? normalized(input.generatedDescription ?? null) : current.generatedDescription, input.sensitive, + nextSensitivityReason, ); if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." }); return updated; diff --git a/backend/src/routes/workspaces.ts b/backend/src/routes/workspaces.ts index a2b4a563..981ac7f5 100644 --- a/backend/src/routes/workspaces.ts +++ b/backend/src/routes/workspaces.ts @@ -16,23 +16,33 @@ import { type WorkspaceDescriptor, } from "../workspaces/schema.js"; import type { RuntimeBindings } from "../workspaces/runtime-renderer.js"; -import type { ConnectorDiagnostics } from "../workspaces/diagnostics.js"; +import type { + ConnectorDiagnostics, + Diagnostic, + WorkspaceDiagnosticOptions, +} from "../workspaces/diagnostics.js"; import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; import type { AuthDiagnoser } from "../auth/diagnostics.js"; import { decodeAuthDiagnostics, type AuthDiagnostics } from "../auth/group-catalog.js"; +import type { WorkspaceDatabase } from "../catalog/types.js"; export type WorkspaceDiagnoser = ( workspace: WorkspaceDescriptor, bindings: RuntimeBindings, - options: { writeProbe: boolean }, + options: WorkspaceDiagnosticOptions, ) => Promise; +export type WorkspaceDatabaseTester = ( + workspaceId: string, +) => Promise; + interface WorkspaceRoutesDeps { registry: WorkspaceRegistry; config: WorkspaceRegistryConfig; diagnose: WorkspaceDiagnoser; authDiagnoser: AuthDiagnoser; secretStore: WorkspaceSecretStore; + testDatabaseConnection: WorkspaceDatabaseTester; } const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/); @@ -56,6 +66,20 @@ const SAFE_MESSAGES = { semantic_index_incompatible: "Semantic index is incompatible with this workspace.", } as const; +const catalogConnectionUnavailable = (): Diagnostic => ({ + level: "error", + code: "connector_unavailable", + field: "dwh", + message: "The configured database could not be reached or authenticated.", +}); + +const catalogConnectionMissing = (): Diagnostic => ({ + level: "error", + code: "binding_missing", + field: "dwh", + message: "Configure this workspace in Database Management before testing connections.", +}); + function authenticationReport(value: unknown): AuthDiagnostics { const report = decodeAuthDiagnostics(value); if (!report) throw new Error("invalid authentication diagnostic report"); @@ -238,14 +262,33 @@ export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps) deps.secretStore, ); try { - const [workspaceDiagnostics, inspectedAuthentication] = await Promise.all([ - deps.diagnose(operational, lease.bindings, { writeProbe: false }), + const [workspaceDiagnostics, testedDatabase, inspectedAuthentication] = await Promise.all([ + deps.diagnose(operational, lease.bindings, { + writeProbe: false, + skipDwh: true, + }), + deps.testDatabaseConnection(id), deps.authDiagnoser.inspect({ live: true }), ]); const authentication = authenticationReport(inspectedAuthentication); + const catalogConnectionReady = testedDatabase?.connectionStatus === "reachable"; + const catalogConnectionDiagnostic = !testedDatabase + ? catalogConnectionMissing() + : catalogConnectionReady + ? undefined + : catalogConnectionUnavailable(); + const diagnostics = catalogConnectionDiagnostic + ? [ + ...workspaceDiagnostics.diagnostics.filter(({ code }) => code !== "binding_ok"), + catalogConnectionDiagnostic, + ] + : workspaceDiagnostics.diagnostics; return { ...workspaceDiagnostics, - activatable: workspaceDiagnostics.activatable && authentication.ready, + activatable: workspaceDiagnostics.activatable + && catalogConnectionReady + && authentication.ready, + diagnostics, authentication, }; } finally { diff --git a/backend/src/workspaces/diagnostics.ts b/backend/src/workspaces/diagnostics.ts index fdf4fc90..ff122193 100644 --- a/backend/src/workspaces/diagnostics.ts +++ b/backend/src/workspaces/diagnostics.ts @@ -130,6 +130,11 @@ export interface DiagnosticAdapters { probeEmbedding(request: EmbeddingDiagnosticRequest): Promise; } +export interface WorkspaceDiagnosticOptions { + writeProbe: boolean; + skipDwh?: boolean; +} + export const DEFAULT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS = 5_000; async function secretPresent(file: string): Promise { @@ -421,6 +426,7 @@ async function diagnoseValidatedWorkspace( adapters: DiagnosticAdapters, timeoutMs: number, semanticRuntime: SemanticRuntimeConfig, + skipDwh: boolean, ): Promise { const evidenceField = descriptor.evidence?.source.type === "http" ? "evidence.source.authentication" @@ -432,75 +438,79 @@ async function diagnoseValidatedWorkspace( variable, })); const diagnostics = [ - ...[...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field)), + ...(skipDwh + ? [] + : [...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field))), ...evidenceDiagnostics, ]; if (diagnostics.length > 0) return { activatable: false, diagnostics }; - const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs); let activatable = true; - const dwhValues = bindings.dwh.values; - const dwhField = (suffix: string) => bindingName(descriptor, suffix); - const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema }; - let dwhRequest: ConnectorDiagnosticRequest | undefined; - if (bindings.dwh.transport === "rest_api") { - const diagnostic = descriptor.diagnostics?.dwh_rest; - const baseUrl = dwhValues[dwhField("BASE_URL")]; - if (diagnostic && baseUrl) { - const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")]; - if (diagnostic.auth === "none" || credentialFile !== undefined) { + if (!skipDwh) { + const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs); + const dwhValues = bindings.dwh.values; + const dwhField = (suffix: string) => bindingName(descriptor, suffix); + const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema }; + let dwhRequest: ConnectorDiagnosticRequest | undefined; + if (bindings.dwh.transport === "rest_api") { + const diagnostic = descriptor.diagnostics?.dwh_rest; + const baseUrl = dwhValues[dwhField("BASE_URL")]; + if (diagnostic && baseUrl) { + const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")]; + if (diagnostic.auth === "none" || credentialFile !== undefined) { + dwhRequest = { + role: "dwh", + transport: "rest_api", + baseUrl, + credentialFile, + tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")], + resource: dwhResource, + timeoutMs: dwhTimeout, + signal: new AbortController().signal, + diagnostic, + }; + } + } + } else if (bindings.dwh.transport === "postgres_direct") { + const host = dwhValues[dwhField("HOST")]; + const port = numericBinding(dwhValues, dwhField("PORT")); + const user = dwhValues[dwhField("USER")]; + const credentialFile = dwhValues[dwhField("PASSWORD_FILE")]; + if (host && port && user && credentialFile) { dwhRequest = { role: "dwh", - transport: "rest_api", - baseUrl, + transport: "postgres_direct", + host, + port, + user, credentialFile, tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")], resource: dwhResource, timeoutMs: dwhTimeout, signal: new AbortController().signal, - diagnostic, }; } } - } else if (bindings.dwh.transport === "postgres_direct") { - const host = dwhValues[dwhField("HOST")]; - const port = numericBinding(dwhValues, dwhField("PORT")); - const user = dwhValues[dwhField("USER")]; - const credentialFile = dwhValues[dwhField("PASSWORD_FILE")]; - if (host && port && user && credentialFile) { - dwhRequest = { - role: "dwh", - transport: "postgres_direct", - host, - port, - user, - credentialFile, - tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")], - resource: dwhResource, - timeoutMs: dwhTimeout, - signal: new AbortController().signal, - }; + + if (!dwhRequest) { + diagnostics.push(diagnosticError("workspace_not_activatable")); + return { activatable: false, diagnostics }; } - } - if (!dwhRequest) { - diagnostics.push(diagnosticError("workspace_not_activatable")); - return { activatable: false, diagnostics }; - } - - try { - const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({ - ...dwhRequest, - signal, - timeoutMs: dwhTimeout, - })); - if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) { + try { + const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({ + ...dwhRequest, + signal, + timeoutMs: dwhTimeout, + })); + if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + } catch { diagnostics.push(diagnosticError("connector_unavailable")); activatable = false; } - } catch { - diagnostics.push(diagnosticError("connector_unavailable")); - activatable = false; } try { @@ -571,11 +581,18 @@ export function createWorkspaceDiagnoser( return async function diagnose( workspace: WorkspaceDescriptor, bindings: RuntimeBindings, - _options: { writeProbe: boolean }, + diagnosticOptions: WorkspaceDiagnosticOptions, ): Promise { requireSupportedDescriptor(workspace); const descriptor = validateWorkspaceDescriptor(workspace); - return await diagnoseValidatedWorkspace(descriptor, bindings, adapters, timeoutMs, semanticRuntime); + return await diagnoseValidatedWorkspace( + descriptor, + bindings, + adapters, + timeoutMs, + semanticRuntime, + diagnosticOptions.skipDwh ?? false, + ); }; } diff --git a/backend/test/catalog-databases-routes.test.ts b/backend/test/catalog-databases-routes.test.ts index 4648b002..198329d6 100644 --- a/backend/test/catalog-databases-routes.test.ts +++ b/backend/test/catalog-databases-routes.test.ts @@ -11,6 +11,7 @@ import type { ObservedSchemaSnapshot } from "../src/catalog/types.js"; import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js"; import type { WorkspaceDescriptor } from "../src/workspaces/schema.js"; +import type { WorkspaceDiagnoser } from "../src/routes/workspaces.js"; const roots: string[] = []; afterEach(() => { @@ -33,6 +34,7 @@ function setup( catalogDependencies: { catalogOperationCoordinator?: CatalogOperationCoordinator; catalogPostgresAccess?: CatalogPostgresAccess; + workspaceDiagnoser?: WorkspaceDiagnoser; } = {}, workspaceDescriptor: WorkspaceDescriptor = workspace, ) { @@ -56,7 +58,7 @@ function setup( workspaceRegistry: registry, workspaceSecretStore: secretStore, catalogRepository: repository, - workspaceDiagnoser: vi.fn(), + workspaceDiagnoser: vi.fn(async () => ({ activatable: true, diagnostics: [] })), ...catalogDependencies, }); return { app, secretStore, repository }; @@ -275,6 +277,72 @@ test("rejects a connection test while another catalog operation owns the databas } }); +test("workspace and database tests use the same current catalog database binding", async () => { + const connect = vi.fn(async () => ({ + query: vi.fn(async () => ({ + rows: [{ database: "warehouse", schema: "datawarehouse" }], + })), + end: vi.fn(async () => undefined), + })); + const diagnose: WorkspaceDiagnoser = vi.fn(async () => ({ + activatable: true, + diagnostics: [{ + level: "info", + code: "binding_ok", + message: "Installation bindings and diagnostics succeeded.", + }], + })); + const { app } = setup({ + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + THT_WS_PSD_CLINICAL_DWH_HOST: "legacy-db.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "legacy-reader", + }, { + catalogPostgresAccess: { connect } as CatalogPostgresAccess, + workspaceDiagnoser: diagnose, + }); + const created = (await app.inject({ + method: "POST", + url: "/catalog/databases", + payload: { + ...direct, + binding: { ...direct.binding, host: "current-db.internal", username: "current-reader" }, + }, + })).json(); + + const databaseTest = await app.inject({ + method: "POST", + url: `/catalog/databases/${created.id}/test`, + payload: { version: created.version }, + }); + const workspaceTest = await app.inject({ + method: "POST", + url: "/workspaces/psd-clinical/test", + payload: {}, + }); + + expect(databaseTest.statusCode).toBe(200); + expect(workspaceTest.statusCode).toBe(200); + expect(connect).toHaveBeenCalledTimes(2); + expect(connect.mock.calls.map(([database]) => database)).toEqual([ + expect.objectContaining({ + databaseName: "warehouse", + schema: "datawarehouse", + binding: expect.objectContaining({ host: "current-db.internal", username: "current-reader" }), + }), + expect.objectContaining({ + databaseName: "warehouse", + schema: "datawarehouse", + binding: expect.objectContaining({ host: "current-db.internal", username: "current-reader" }), + }), + ]); + expect(diagnose).toHaveBeenCalledWith( + workspace, + expect.any(Object), + { writeProbe: false, skipDwh: true }, + ); +}); + test("returns exact global and per-database fleet metrics", async () => { const { app, repository } = setup(); const database = await repository.create(direct); diff --git a/backend/test/catalog-description-generation-routes.test.ts b/backend/test/catalog-description-generation-routes.test.ts index ff8b395e..6006efa1 100644 --- a/backend/test/catalog-description-generation-routes.test.ts +++ b/backend/test/catalog-description-generation-routes.test.ts @@ -167,7 +167,7 @@ test("assesses sensitive flags locally without persisting them or calling an LLM scope: "all", engine: "local", modelId: null, - policyVersion: "sensitivity-v2", + policyVersion: "sensitivity-v4", status: "completed", total: 1, suggestedSensitive: 1, @@ -224,12 +224,30 @@ test("assesses sensitive flags locally without persisting them or calling an LLM runId: responseBody.run.id, sequence: 2, level: "info", - message: "Assessed 1 of 1 columns locally.", + message: "Scanning source data: pass 1 of 3, table batch 1 of 1.", }, { runId: responseBody.run.id, sequence: 3, level: "info", + message: "Scanning source data: pass 2 of 3, table batch 1 of 1.", + }, + { + runId: responseBody.run.id, + sequence: 4, + level: "info", + message: "Scanning source data: pass 3 of 3, table batch 1 of 1.", + }, + { + runId: responseBody.run.id, + sequence: 5, + level: "info", + message: "Assessed 1 of 1 columns locally.", + }, + { + runId: responseBody.run.id, + sequence: 6, + level: "info", message: "Local sensitivity analysis completed for 1 column.", }, ]); diff --git a/backend/test/catalog-description-generation.integration.test.ts b/backend/test/catalog-description-generation.integration.test.ts index cb23bf1b..da868e56 100644 --- a/backend/test/catalog-description-generation.integration.test.ts +++ b/backend/test/catalog-description-generation.integration.test.ts @@ -16,6 +16,7 @@ import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_s import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js"; import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js"; import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js"; +import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js"; import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js"; import { loadConfig } from "../src/config.js"; import type { WorkspaceRegistry } from "../src/workspaces/registry.js"; @@ -54,6 +55,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a await upAiTokenUsage(db); await upCanonicalModelIds(db); await upLocalSensitivityAnalysis(db); + await upSensitivityReason(db); const repository = new KyselyCatalogRepository(db); const database = await repository.create({ workspaceId: "psd-clinical", diff --git a/backend/test/catalog-repository.integration.test.ts b/backend/test/catalog-repository.integration.test.ts index 390a8091..8cb3f36b 100644 --- a/backend/test/catalog-repository.integration.test.ts +++ b/backend/test/catalog-repository.integration.test.ts @@ -17,6 +17,7 @@ import { up as upLogicalRelationships } from "../src/catalog/migrations/008_cata import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js"; import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js"; import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js"; +import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js"; const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0; @@ -56,6 +57,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo }).execute(); await upCanonicalModelIds(db); await upLocalSensitivityAnalysis(db); + await upSensitivityReason(db); await expect(db.selectFrom("sensitiveDataSuggestionRuns") .select(["engine", "modelId", "policyVersion", "unknown"]) .where("id", "=", historicalSuggestionRunId) @@ -142,7 +144,11 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo patientName.description, patientName.generatedDescription, true, - )).toMatchObject({ sensitive: true }); + "Local assessment matched content rule pii.person_name.", + )).toMatchObject({ + sensitive: true, + sensitivityReason: "Local assessment matched content rule pii.person_name.", + }); expect(await repository.getCatalogMetrics(created.id)).toEqual({ scope: "database", databaseId: created.id, @@ -187,6 +193,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo )).toMatchObject({ updated: 1 }); expect(await repository.getColumn(created.id, patients.id, patientName.id)).toMatchObject({ sensitive: true, + sensitivityReason: "Local assessment matched content rule pii.person_name.", sourceComment: "Sensitive patient name", }); @@ -281,10 +288,33 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl }; await repository.applySchemaSync(database.id, database.version, "all", [], snapshot); const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!; + const context = (await repository.getLogicalRelationshipContext(database.id))!; + const source = context.endpoints.find((endpoint) => ( + endpoint.tableName === "visits" && endpoint.columnName === "id" + ))!; + const target = context.endpoints.find((endpoint) => ( + endpoint.tableName === "patients" && endpoint.columnName === "id" + ))!; + const generatedCandidate = { sourceColumnId: source.columnId, targetColumnId: target.columnId }; - expect(await repository.deleteTableMetadata(database.id, [patients.id], "relationships")) - .toEqual({ tables: 0, columns: 0, relationships: 1 }); + await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate])) + .resolves.toBe(1); + await expect(repository.getCatalogMetrics(database.id)) + .resolves.toMatchObject({ relationships: 2 }); + expect(await repository.deleteDatabaseMetadata([database.id], "relationships")) + .toEqual({ tables: 0, columns: 0, relationships: 2 }); expect(await repository.listRelationships(database.id)).toEqual([]); + expect(await repository.listLogicalRelationships(database.id)).toEqual([]); + await expect(repository.getCatalogMetrics(database.id)) + .resolves.toMatchObject({ relationships: 0 }); + + await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot); + await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate])) + .resolves.toBe(1); + expect(await repository.deleteTableMetadata(database.id, [patients.id], "relationships")) + .toEqual({ tables: 0, columns: 0, relationships: 2 }); + expect(await repository.listRelationships(database.id)).toEqual([]); + expect(await repository.listLogicalRelationships(database.id)).toEqual([]); expect(await repository.listColumns(database.id, patients.id)).toHaveLength(2); expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined(); @@ -298,13 +328,24 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl await repository.applySchemaSync(database.id, database.version, "columns", [patients.id], snapshot); await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot); + const refreshedContext = (await repository.getLogicalRelationshipContext(database.id))!; + const refreshedSource = refreshedContext.endpoints.find((endpoint) => ( + endpoint.tableName === "visits" && endpoint.columnName === "id" + ))!; + const refreshedTarget = refreshedContext.endpoints.find((endpoint) => ( + endpoint.tableName === "patients" && endpoint.columnName === "id" + ))!; + await expect(repository.insertGeneratedLogicalRelationships(database.id, [{ + sourceColumnId: refreshedSource.columnId, + targetColumnId: refreshedTarget.columnId, + }])).resolves.toBe(1); expect(await repository.deleteDatabaseMetadata([ database.id, "99999999-9999-4999-8999-999999999999", ], "tables")).toBeUndefined(); expect(await repository.listTables(database.id)).toHaveLength(2); expect(await repository.deleteDatabaseMetadata([database.id], "tables")) - .toEqual({ tables: 2, columns: 4, relationships: 1 }); + .toEqual({ tables: 2, columns: 4, relationships: 2 }); expect(await repository.get(database.id)).toBeDefined(); expect(await repository.listTables(database.id)).toEqual([]); expect(await repository.listRelationships(database.id)).toEqual([]); @@ -431,6 +472,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se await upAiTokenUsage(db); await upCanonicalModelIds(db); await upLocalSensitivityAnalysis(db); + await upSensitivityReason(db); const repository = new KyselyCatalogRepository(db); const firstDatabase = await repository.create({ workspaceId: "generation-one", @@ -703,6 +745,8 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists logical relationsh const target = context.endpoints.find((item) => item.tableName === "users" && item.columnName === "id")!; const created = await repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, false); expect(created).toMatchObject({ origin: "manual", status: "active" }); + expect(await repository.getCatalogMetrics(database.id)) + .toMatchObject({ relationships: 1 }); await expect(repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, true)) .resolves.toBeUndefined(); diff --git a/backend/test/catalog-schema-routes.test.ts b/backend/test/catalog-schema-routes.test.ts index 66cc9069..9b1dd595 100644 --- a/backend/test/catalog-schema-routes.test.ts +++ b/backend/test/catalog-schema-routes.test.ts @@ -7,7 +7,11 @@ import { loadConfig } from "../src/config.js"; import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js"; import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js"; import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js"; -import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js"; +import { + CatalogConnectorError, + type CatalogSyncRun, + type ObservedSchemaSnapshot, +} from "../src/catalog/types.js"; import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js"; import type { WorkspaceDescriptor } from "../src/workspaces/schema.js"; @@ -164,6 +168,32 @@ test("synchronizes a full physical schema and derives primary and foreign key fl expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version); }); +test("attempts synchronization after a failed connection test and reports the live access failure", async () => { + const { app, repository, database, scan } = await setup(); + await repository.recordTest(database.id, database.version, { + connectionStatus: "failed", + testedVersion: database.version, + lastTestedAt: new Date().toISOString(), + lastErrorCode: "connector_unavailable", + lastErrorMessage: "The database connector could not be reached or authenticated.", + }); + scan.mockRejectedValueOnce(new CatalogConnectorError("upstream credentials must not escape")); + + const started = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/sync-runs`, + payload: { version: database.version, scope: "all", tableIds: [] }, + }); + + expect(started.statusCode).toBe(202); + const failed = await waitFor(repository, started.json().id, "failed"); + expect(scan).toHaveBeenCalledOnce(); + expect(failed).toMatchObject({ + errorCode: "schema_introspection_failed", + errorMessage: "The database schema could not be read. Check the connection and credentials, then try again.", + }); +}); + test("synchronizes columns for every catalog table when no table selection is supplied", async () => { const { app, repository, database, setObserved } = await setup(); const tablesRun = await app.inject({ @@ -247,13 +277,18 @@ test("keeps generated descriptions editable and preserves them across synchroniz }); const sensitiveOnly = await app.inject({ method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`, - payload: { version: editedColumn.json().version, sensitive: true }, + payload: { + version: editedColumn.json().version, + sensitive: true, + sensitivityReason: "Local assessment matched content rule pii.email.", + }, }); expect(sensitiveOnly.statusCode).toBe(200); expect(sensitiveOnly.json()).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft", sensitive: true, + sensitivityReason: "Local assessment matched content rule pii.email.", }); const emptyPatch = await app.inject({ method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`, @@ -268,6 +303,7 @@ test("keeps generated descriptions editable and preserves them across synchroniz description: "Reviewed key", generatedDescription: "Generated key draft", sensitive: true, + sensitivityReason: "Local assessment matched content rule pii.email.", }); }); @@ -358,6 +394,67 @@ test("consolidates non-empty generated column descriptions and preserves curated expect(scan).not.toHaveBeenCalled(); }); +test("consolidates generated descriptions for every column in a database", async () => { + const { app, repository, database, scan } = await setup(); + await seedCatalog(repository, database); + const tables = await repository.listTables(database.id); + const patients = tables.find((table) => table.name === "patients")!; + const visits = tables.find((table) => table.name === "visits")!; + const patientId = (await repository.listColumns(database.id, patients.id))[0]!; + const visitColumns = await repository.listColumns(database.id, visits.id); + const visitId = visitColumns.find((column) => column.name === "id")!; + const visitPatientId = visitColumns.find((column) => column.name === "patient_id")!; + await repository.updateColumnMetadata( + database.id, + patients.id, + patientId.id, + patientId.version, + "Curated patient identifier", + "Generated patient identifier", + ); + await repository.updateColumnMetadata( + database.id, + visits.id, + visitId.id, + visitId.version, + "Curated visit identifier", + "Generated visit identifier", + ); + await repository.updateColumnMetadata( + database.id, + visits.id, + visitPatientId.id, + visitPatientId.version, + "Keep curated patient reference", + "", + ); + + const response = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/descriptions/consolidate`, + payload: { target: "database_columns" }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual({ copied: 2, skipped: 1 }); + expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({ + description: "Generated patient identifier", + generatedDescription: "Generated patient identifier", + version: patientId.version + 2, + }); + expect(await repository.getColumn(database.id, visits.id, visitId.id)).toMatchObject({ + description: "Generated visit identifier", + generatedDescription: "Generated visit identifier", + version: visitId.version + 2, + }); + expect(await repository.getColumn(database.id, visits.id, visitPatientId.id)).toMatchObject({ + description: "Keep curated patient reference", + generatedDescription: "", + version: visitPatientId.version + 1, + }); + expect(scan).not.toHaveBeenCalled(); +}); + test("rejects description consolidation while the Workspace Database is reserved", async () => { const { app, repository, database, operations } = await setup(); await seedCatalog(repository, database); @@ -432,9 +529,14 @@ test("strictly validates description consolidation database and target ids", asy url: `/catalog/databases/${database.id}/descriptions/consolidate`, payload: { target: "tables", targetIds: [table.id], unexpected: true }, }), + app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/descriptions/consolidate`, + payload: { target: "database_columns", targetIds: [table.id] }, + }), ]); - expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400]); + expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400]); for (const response of responses) { expect(response.json()).toEqual({ code: "description_consolidation_invalid", @@ -524,6 +626,18 @@ test("deletes relationships for selected databases without deleting their tables const { app, repository, database } = await setup(); await seedCatalog(repository, database); const tables = await repository.listTables(database.id); + const context = (await repository.getLogicalRelationshipContext(database.id))!; + const source = context.endpoints.find((endpoint) => ( + endpoint.tableName === "visits" && endpoint.columnName === "id" + ))!; + const target = context.endpoints.find((endpoint) => ( + endpoint.tableName === "patients" && endpoint.columnName === "id" + ))!; + await expect(repository.insertGeneratedLogicalRelationships(database.id, [{ + sourceColumnId: source.columnId, + targetColumnId: target.columnId, + }])).resolves.toBe(1); + await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 2 }); const response = await app.inject({ method: "POST", @@ -532,10 +646,12 @@ test("deletes relationships for selected databases without deleting their tables }); expect(response.statusCode).toBe(200); - expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 1 }); + expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 2 }); expect(await repository.listTables(database.id)).toHaveLength(2); expect(await repository.listColumns(database.id, tables[0]!.id)).not.toEqual([]); expect(await repository.listRelationships(database.id)).toEqual([]); + expect(await repository.listLogicalRelationships(database.id)).toEqual([]); + await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 0 }); expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined(); }); @@ -671,6 +787,11 @@ test("rebuilds generated relationships and returns the exact summary", async () }); expect(first.statusCode).toBe(200); expect(first.json()).toEqual({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 }); + const metrics = await app.inject({ + method: "GET", url: `/catalog/metrics?databaseId=${database.id}`, + }); + expect(metrics.statusCode).toBe(200); + expect(metrics.json()).toMatchObject({ relationships: 1 }); const generated = (await repository.listLogicalRelationships(database.id))[0]!; await app.inject({ diff --git a/backend/test/catalog-sensitivity-analysis.test.ts b/backend/test/catalog-sensitivity-analysis.test.ts index f441f486..5e9f49f6 100644 --- a/backend/test/catalog-sensitivity-analysis.test.ts +++ b/backend/test/catalog-sensitivity-analysis.test.ts @@ -73,7 +73,7 @@ const running: SensitivityAnalysisRun = { scope: "all", engine: "local", modelId: null, - policyVersion: "sensitivity-v2", + policyVersion: "sensitivity-v4", status: "running", total: 0, suggestedSensitive: 0, @@ -132,27 +132,36 @@ test("classifies all selected tables in one breadth-first run and reports covera tableId === firstTable.id ? [firstColumn] : [secondColumn] )), } as unknown as CatalogRepository; - const assess = vi.fn(async () => [ - { - columnId: firstColumn.id, - assessment: "non_sensitive" as const, - proposedSensitive: false, - evidence: [{ kind: "coverage" as const, ruleId: "coverage.sampled_1000" }], - observedValues: 1_000, - coverage: "sampled" as const, - }, - { - columnId: secondColumn.id, - assessment: "sensitive" as const, - proposedSensitive: true, - evidence: [{ kind: "content" as const, ruleId: "pii.email" }], - observedValues: 12, - coverage: "sampled" as const, - }, - ]); + const assess = vi.fn(async ( + _targets, + _signal, + _nerBudget, + onActivity?: (message: string) => void | Promise, + ) => { + await onActivity?.("Scanning source data: pass 1 of 3, table batch 1 of 1."); + return [ + { + columnId: firstColumn.id, + assessment: "non_sensitive" as const, + proposedSensitive: false, + evidence: [{ kind: "coverage" as const, ruleId: "coverage.sampled_1000" }], + observedValues: 1_000, + coverage: "sampled" as const, + }, + { + columnId: secondColumn.id, + assessment: "sensitive" as const, + proposedSensitive: true, + evidence: [{ kind: "content" as const, ruleId: "pii.email" }], + observedValues: 12, + coverage: "sampled" as const, + }, + ]; + }); const classifier = { assess } as unknown as SensitivityClassifier; const onPrepared = vi.fn(); const onProgress = vi.fn(); + const onActivity = vi.fn(); const suggestions = await new SensitivityAnalysisService(repository, classifier).analyze( database.id, @@ -161,6 +170,7 @@ test("classifies all selected tables in one breadth-first run and reports covera new AbortController().signal, onPrepared, onProgress, + onActivity, ); expect(assess).toHaveBeenCalledOnce(); @@ -169,6 +179,9 @@ test("classifies all selected tables in one breadth-first run and reports covera { database, table: secondTable, columns: [secondColumn] }, ]); expect(onPrepared).toHaveBeenCalledWith(2); + expect(onActivity).toHaveBeenCalledWith( + "Scanning source data: pass 1 of 3, table batch 1 of 1.", + ); expect(onProgress.mock.calls.map(([processed]) => processed)).toEqual([1, 2]); expect(suggestions).toEqual([ expect.objectContaining({ columnId: firstColumn.id, sensitive: false, coverage: "sampled" }), @@ -176,6 +189,52 @@ test("classifies all selected tables in one breadth-first run and reports covera ]); }); +test("persists classifier activity in the running analysis event log", async () => { + let persisted = running; + const appendEvent = vi.fn(async () => undefined); + const repository = { + get: vi.fn(async () => database), + createSensitivityAnalysisRun: vi.fn(async () => running), + getSensitivityAnalysisRun: vi.fn(async () => persisted), + updateSensitivityAnalysisRun: vi.fn(async ( + _runId: string, + changes: Partial, + ) => { + persisted = { ...persisted, ...changes }; + return persisted; + }), + appendSensitivityAnalysisEvent: appendEvent, + } as unknown as CatalogRepository; + const analysis = { + analyze: vi.fn(async ( + _databaseId, + _scope, + _targetIds, + _signal, + onPrepared, + _onProgress, + onActivity, + ) => { + await onPrepared?.(0); + await onActivity?.("Scanning source data: pass 1 of 3, table batch 1 of 1."); + return []; + }), + } as unknown as SensitivityAnalysisService; + + await new SensitivityAnalysisRunner(repository, analysis).run( + database.id, + "all", + [], + new AbortController().signal, + ); + + expect(appendEvent).toHaveBeenCalledWith( + running.id, + "info", + "Scanning source data: pass 1 of 3, table batch 1 of 1.", + ); +}); + test("marks a created run interrupted if the request deadline expires during persistence", async () => { const controller = new AbortController(); const update = vi.fn(async (_runId: string, changes: Partial) => ({ diff --git a/backend/test/catalog-sensitivity-classifier.test.ts b/backend/test/catalog-sensitivity-classifier.test.ts index 3692ab75..ff9abe83 100644 --- a/backend/test/catalog-sensitivity-classifier.test.ts +++ b/backend/test/catalog-sensitivity-classifier.test.ts @@ -69,6 +69,32 @@ function source(scan: SensitivityTableScan): SensitivityValueSource { }) }; } +test("reports source-scan activity before a long table scan completes", async () => { + let releaseScan!: () => void; + const scanGate = new Promise((resolve) => { + releaseScan = resolve; + }); + const scanTable = vi.fn(async () => { + await scanGate; + return { kind: "complete" as const, observedValues: 0 }; + }); + const activity = vi.fn(); + const analysis = new SensitivityClassifier({ scanTable }).assess( + [{ database, table, columns: [column()] }], + new AbortController().signal, + undefined, + activity, + ); + + await vi.waitFor(() => expect(scanTable).toHaveBeenCalledOnce()); + releaseScan(); + await analysis; + + expect(activity).toHaveBeenCalledWith( + "Scanning source data: pass 1 of 3, table batch 1 of 1.", + ); +}); + test("one email hidden in a generically named column makes the whole column sensitive", async () => { const target = column(); const values = source({ @@ -351,6 +377,98 @@ test("strong Italian PII metadata is sensitive even when the source column is em expect(values.scanTable).not.toHaveBeenCalled(); }); +test("excludes bigint primary keys from content analysis as non-informative identifiers", async () => { + const target = column({ + name: "id", + dataType: "bigint", + primaryKeyPosition: 1, + isPrimaryKey: true, + }); + const values = source({ + batches: [[{ + columnId: target.id, + value: "3471234567", + characterLength: 10, + }]], + coverage: { kind: "complete", observedValues: 1 }, + }); + + const [assessment] = await new SensitivityClassifier(values).assessTable( + { database, table, columns: [target] }, + new AbortController().signal, + ); + + expect(assessment).toMatchObject({ + assessment: "non_sensitive", + proposedSensitive: false, + evidence: [{ + kind: "type", + ruleId: "type.bigint_primary_key_non_informative", + label: "non-informative bigint primary key", + }], + coverage: "metadata", + }); + expect(values.scanTable).not.toHaveBeenCalled(); +}); + +test("infers an undeclared bigint column named pk as a non-informative primary-key identifier", async () => { + const target = column({ + name: "pk", + dataType: "bigint", + primaryKeyPosition: null, + isPrimaryKey: false, + }); + const values = source({ + batches: [[{ + columnId: target.id, + value: "3471234567", + characterLength: 10, + }]], + coverage: { kind: "complete", observedValues: 1 }, + }); + + const [assessment] = await new SensitivityClassifier(values).assessTable( + { database, table, columns: [target] }, + new AbortController().signal, + ); + + expect(assessment).toMatchObject({ + assessment: "non_sensitive", + proposedSensitive: false, + evidence: [{ + kind: "metadata", + ruleId: "metadata.bigint_pk_identifier_non_informative", + label: "non-informative conventional bigint primary-key identifier", + }], + coverage: "metadata", + }); + expect(values.scanTable).not.toHaveBeenCalled(); +}); + +test("still inspects phone-like values in bigint columns that are not primary keys", async () => { + const target = column({ name: "id", dataType: "bigint" }); + const values = source({ + batches: [[{ + columnId: target.id, + value: "3471234567", + characterLength: 10, + }]], + coverage: { kind: "complete", observedValues: 1 }, + }); + + const [assessment] = await new SensitivityClassifier(values).assessTable( + { database, table, columns: [target] }, + new AbortController().signal, + ); + + expect(assessment).toMatchObject({ + assessment: "sensitive", + proposedSensitive: true, + evidence: [{ kind: "content", ruleId: "pii.phone_number" }], + }); + expect(values.scanTable).toHaveBeenCalledOnce(); +}); + test.each([ ["RSSMRA85T10A562S", "pii.italian_fiscal_code"], ["IT60 X054 2811 1010 0000 0123 456", "financial.iban"], diff --git a/backend/test/catalog-tables-routes.test.ts b/backend/test/catalog-tables-routes.test.ts index 7ac26acd..d230c104 100644 --- a/backend/test/catalog-tables-routes.test.ts +++ b/backend/test/catalog-tables-routes.test.ts @@ -103,7 +103,7 @@ test("requires an exact deletion confirmation before applying the atomic diff", ]); }); -test("refuses synchronization until the current binding has passed its connection test", async () => { +test("starts synchronization without requiring a prior connection test", async () => { const { app, repository, database } = await setup(); await repository.update(database.id, database.version, { workspaceId: database.workspaceId, @@ -117,6 +117,10 @@ test("refuses synchronization until the current binding has passed its connectio url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version + 1, scope: "tables", tableIds: [] }, }); - expect(response.statusCode).toBe(409); - expect(response.json()).toMatchObject({ code: "schema_sync_conflict" }); + expect(response.statusCode).toBe(202); + expect(response.json()).toMatchObject({ + databaseId: database.id, + scope: "tables", + state: "queued", + }); }); diff --git a/backend/test/routes-workspaces.test.ts b/backend/test/routes-workspaces.test.ts index fd28f0a7..9140cef3 100644 --- a/backend/test/routes-workspaces.test.ts +++ b/backend/test/routes-workspaces.test.ts @@ -11,6 +11,8 @@ import { WorkspaceRegistry, type WorkspaceRevision } from "../src/workspaces/reg import { serializeWorkspaceYaml, type CanonicalWorkspace } from "../src/workspaces/schema.js"; import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; import type { AuthDiagnoser, AuthDiagnostics } from "../src/auth/diagnostics.js"; +import type { WorkspaceDatabase } from "../src/catalog/types.js"; +import type { WorkspaceDatabaseTester } from "../src/routes/workspaces.js"; const workspace: CanonicalWorkspace = { workspace: { @@ -63,12 +65,35 @@ const readyAuthentication: AuthDiagnostics = { checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }], }; +const reachableWorkspaceDatabase: WorkspaceDatabase = { + id: "db-psd-clinical", + workspaceId: "psd-clinical", + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + version: 1, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + binding: { + transport: "postgres_direct", + host: "current-db.internal", + port: 5432, + username: "current-reader", + }, + connectionStatus: "reachable", + testedVersion: 1, + lastTestedAt: "2026-01-01T00:00:00.000Z", +}; + function appFor( registry: RegistryFake, diagnose = vi.fn(async () => ({ activatable: true, diagnostics: [] })), secretStore = testSecretStore(), env: Record = {}, authDiagnoser: AuthDiagnoser = { inspect: vi.fn(async () => readyAuthentication) }, + workspaceDatabaseTester: WorkspaceDatabaseTester = vi.fn( + async () => reachableWorkspaceDatabase, + ), ) { return buildApp(loadConfig({ THT_HARNESS_DIR: "/missing-harness", @@ -80,6 +105,7 @@ function appFor( workspaceDiagnoser: diagnose, workspaceSecretStore: secretStore, authDiagnoser, + workspaceDatabaseTester, } as any); } @@ -283,7 +309,48 @@ test("runs diagnostics for a schema v4 workspace", async () => { expect(diagnose).toHaveBeenCalledWith(workspace, { dwh: expect.objectContaining({ transport: "postgres_direct" }), evidence: { missing: [], values: {} }, - }, { writeProbe: false }); + }, { writeProbe: false, skipDwh: true }); +}); + +test("reports a missing Database Management configuration without using the legacy DWH test", async () => { + const diagnose = vi.fn(async () => ({ + activatable: true, + diagnostics: [{ + level: "info" as const, + code: "binding_ok" as const, + message: "Installation bindings and diagnostics succeeded.", + }], + })); + const workspaceDatabaseTester = vi.fn(async () => undefined); + const app = appFor( + registryFake(), + diagnose, + testSecretStore(), + {}, + { inspect: vi.fn(async () => readyAuthentication) }, + workspaceDatabaseTester, + ); + + const response = await app.inject({ + method: "POST", url: "/workspaces/psd-clinical/test", payload: {}, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + activatable: false, + diagnostics: [{ + level: "error", + code: "binding_missing", + field: "dwh", + message: "Configure this workspace in Database Management before testing connections.", + }], + }); + expect(diagnose).toHaveBeenCalledWith( + workspace, + expect.anything(), + { writeProbe: false, skipDwh: true }, + ); + expect(workspaceDatabaseTester).toHaveBeenCalledWith("psd-clinical"); }); test("reports runtime secret requirements without returning stored values", async () => { diff --git a/backend/test/workspaces-diagnostics.test.ts b/backend/test/workspaces-diagnostics.test.ts index 6ebd09ea..4e86984b 100644 --- a/backend/test/workspaces-diagnostics.test.ts +++ b/backend/test/workspaces-diagnostics.test.ts @@ -89,6 +89,28 @@ test("diagnoses workspace-v4 DWH plus installation-derived Qdrant and Ollama", a })); }); +test("can delegate the DWH probe to Database Management", async () => { + const adapters = successfulAdapters(); + const result = await diagnose(adapters)(workspace, { + dwh: { + ...bindings.dwh, + missing: ["THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"], + }, + evidence: bindings.evidence, + }, { writeProbe: false, skipDwh: true }); + + expect(result).toEqual({ + activatable: true, + diagnostics: [{ + level: "info", code: "binding_ok", + message: "Installation bindings and diagnostics succeeded.", + }], + }); + expect(adapters.probeConnector).not.toHaveBeenCalled(); + expect(adapters.inspectQdrant).toHaveBeenCalledTimes(1); + expect(adapters.probeEmbedding).toHaveBeenCalledTimes(1); +}); + test("reports incompatible internal Qdrant or Ollama metadata", async () => { const vector = await diagnose(successfulAdapters({ inspectQdrant: vi.fn(async () => ({ diff --git a/docs/architecture/components.md b/docs/architecture/components.md index 653b5d11..15c0976a 100644 --- a/docs/architecture/components.md +++ b/docs/architecture/components.md @@ -115,9 +115,12 @@ short text, but it cannot make or persist the decision itself. Each attempt has its own durable run and ordered sanitized events, separate from Description Generation because its lifecycle and counters differ. The run records the local policy version and -aggregate decision counts. Coverage, rule identifiers, proposed flags, source values, NER spans, -and worker diagnostics remain transient. Only an explicit administrator save changes the -human-owned Sensitive Data Flag. +aggregate decision counts. Before each potentially long source-scan batch and local-NER table pass, +the classifier emits a sanitized activity event so the polling progress drawer remains visibly +active while the synchronous analysis request is pending. Coverage, rule identifiers, proposed flags, source values, NER spans, +and worker diagnostics remain transient in run history. Only an explicit administrator save changes +the human-owned Sensitive Data Flag; saving a sensitive result also persists its sanitized +Sensitivity Reason as column Catalog Metadata, while clearing the flag removes that reason. ## Main backend classes diff --git a/docs/architecture/thothii-core-sequence.html b/docs/architecture/thothii-core-sequence.html new file mode 100644 index 00000000..afc320cc --- /dev/null +++ b/docs/architecture/thothii-core-sequence.html @@ -0,0 +1,14896 @@ + + + + + + + Sequenza runtime del core ThothII Diagram + + + + + + + + + + + +
+ +
+
+
+

Sequenza runtime del core ThothII

+
+
+ + + + + + + +
+ + Sequenza runtime del core ThothII + A sequence diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + domanda o decisione gate + + + + + + + + POST /sessions o /response + + + + + + + + sessionNew o sessionShow/reopen + + + + + + + + manifest, artifact e ledger + + + + + + + + createFor + bind; configure + searchPack + + configure(...) e searchPack(...) terminano in parallelo prima dell’avvio del modello. + + + + + + + /nuova-domanda o /riprendi-sessione + + In ripresa usa /riprendi-sessione <id> dopo aver riletto provider, modello e thinking dal manifest. + + + + + + + turno modello; token e tool call + + + + + + + + comandi phase / session / evidence + + + + + + + + legge/scrive artifact e ledger + + Il workspace persistito è la verità; DWH è read-only e Qdrant serve il retrieval. + + + + + + + text_delta / ui_request + + + + + + + + SSE via SseHub: text · info · widget + + + + + + + + testo o widget di revisione + + + + + + + Avvio o ripresa + + + + Turno Pi e workflow persistito + + + + Evento SSE e gate umano + + + + + Revisore · browser · Sequence participant + + + + Revisore + browser + + + + Frontend · React + SSE · Sequence participant + + + + Frontend + React + SSE + + + + Core API · Fastify + SseHub · Sequence participant + + + + Core API + Fastify + SseHub + + + + Runtime · manager + bridge · Sequence participant + + + + Runtime + manager + bridge + + + + Pi + tht-gate · RPC per sessione · Sequence participant + + + + Pi + tht-gate + RPC per sessione + + + + Modello AI · provider configurato · Sequence participant + + + + Modello AI + provider configurato + + + + tht CLI · workflow persistito · Sequence participant + + + + tht CLI + workflow persistito + + + + Stato e dati · workspace + dati · Sequence participant + + + + Stato e dati + workspace + dati + + + + + Legend + + + request + + + + return + + + + default message + + + +

+ + + + + + + + + +
+ + +
+
+
+
+

Bootstrap

+
+
    +
  • • La sessione viene persistita prima dell’avvio di Pi; configurazione e retrieval terminano prima del primo prompt.
    • +
  • • La ripresa rifiuta sessioni finalizzate o archiviate e usa sempre /riprendi-sessione <id>.
    • +
  • • Il modello propone; il revisore decide tramite reviewer_select, reviewer_decide o reviewer_confirm.
    • +
  • • La risposta percorre POST /sessions/:id/response → bridge.respond → extension_ui_response; stale/duplicate ricevono 409.
    • +
  • • La fase avanza solo dopo append della decisione esplicita e fold del ledger persistito.
    • +
+
+
+ +
+ + + + diff --git a/docs/architecture/thothii-core-sequence.visual-check.1440x900.dark.png b/docs/architecture/thothii-core-sequence.visual-check.1440x900.dark.png new file mode 100644 index 00000000..d0473b54 Binary files /dev/null and b/docs/architecture/thothii-core-sequence.visual-check.1440x900.dark.png differ diff --git a/docs/architecture/thothii-core-sequence.visual-check.1440x900.light.png b/docs/architecture/thothii-core-sequence.visual-check.1440x900.light.png new file mode 100644 index 00000000..e6edf510 Binary files /dev/null and b/docs/architecture/thothii-core-sequence.visual-check.1440x900.light.png differ diff --git a/docs/architecture/thothii-core-sequence.visual-check.2048x1320.dark.png b/docs/architecture/thothii-core-sequence.visual-check.2048x1320.dark.png new file mode 100644 index 00000000..b9a48946 Binary files /dev/null and b/docs/architecture/thothii-core-sequence.visual-check.2048x1320.dark.png differ diff --git a/docs/architecture/thothii-core-sequence.visual-check.2048x1320.light.png b/docs/architecture/thothii-core-sequence.visual-check.2048x1320.light.png new file mode 100644 index 00000000..0a97ed49 Binary files /dev/null and b/docs/architecture/thothii-core-sequence.visual-check.2048x1320.light.png differ diff --git a/docs/architecture/thothii-core-sequence.visual-check.html b/docs/architecture/thothii-core-sequence.visual-check.html new file mode 100644 index 00000000..33c14441 --- /dev/null +++ b/docs/architecture/thothii-core-sequence.visual-check.html @@ -0,0 +1,32 @@ + + + + + +Archify automated browser evidence · thothii-core-sequence.html + + + +

Automated browser evidence

thothii-core-sequence.html · visual-check containment pass · perceptual visual review pending

+
+
+ light 1440 by 900 +
LIGHT · 1440×900 · containment pass
+
+
+ dark 1440 by 900 +
DARK · 1440×900 · containment pass
+
+
+ light 2048 by 1320 +
LIGHT · 2048×1320 · containment pass
+
+
+ dark 2048 by 1320 +
DARK · 2048×1320 · containment pass
+
+
+ + diff --git a/docs/architecture/thothii-core-sequence.visual-check.json b/docs/architecture/thothii-core-sequence.visual-check.json new file mode 100644 index 00000000..25b08683 --- /dev/null +++ b/docs/architecture/thothii-core-sequence.visual-check.json @@ -0,0 +1,548 @@ +{ + "schemaVersion": 1, + "ok": true, + "command": "visual-check", + "evidenceKind": "automated-browser", + "status": "pass", + "visualReview": "pending", + "artifact": { + "path": "/Users/mp/projects/ThothII/docs/architecture/thothii-core-sequence.html", + "sha256": "b521eb942f7889cfc3a5e29010546ba59eeab1cf485c22c533d271ccef9a28d5", + "bytes": 716881 + }, + "state": { + "detail": "read", + "motion": "still" + }, + "chrome": { + "status": "available", + "executable": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" + }, + "diagnostics": [], + "containment": { + "status": "pass", + "viewports": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1194, + "diagramWidth": 1164, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1600, + "height": 1000, + "theme": "light", + "innerWidth": 1600, + "innerHeight": 1000, + "scrollWidth": 1600, + "scrollHeight": 1000, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1242, + "diagramWidth": 1212, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1920, + "height": 1080, + "theme": "light", + "innerWidth": 1920, + "innerHeight": 1080, + "scrollWidth": 1920, + "scrollHeight": 1080, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1402, + "diagramWidth": 1372, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1818, + "diagramWidth": 1768, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + } + ] + }, + "readability": { + "status": "pass", + "minimumProjectedNodeTextPx": 6, + "viewports": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1194, + "diagramWidth": 1164, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1600, + "height": 1000, + "theme": "light", + "innerWidth": 1600, + "innerHeight": 1000, + "scrollWidth": 1600, + "scrollHeight": 1000, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1242, + "diagramWidth": 1212, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1920, + "height": 1080, + "theme": "light", + "innerWidth": 1920, + "innerHeight": 1080, + "scrollWidth": 1920, + "scrollHeight": 1080, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1402, + "diagramWidth": 1372, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1818, + "diagramWidth": 1768, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + } + ] + }, + "viewerChrome": { + "status": "pass", + "viewports": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1194, + "diagramWidth": 1164, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1600, + "height": 1000, + "theme": "light", + "innerWidth": 1600, + "innerHeight": 1000, + "scrollWidth": 1600, + "scrollHeight": 1000, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1242, + "diagramWidth": 1212, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1920, + "height": 1080, + "theme": "light", + "innerWidth": 1920, + "innerHeight": 1080, + "scrollWidth": 1920, + "scrollHeight": 1080, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1402, + "diagramWidth": 1372, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1818, + "diagramWidth": 1768, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + } + ] + }, + "captures": { + "status": "pass", + "screenshots": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1194, + "diagramWidth": 1164, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light", + "file": "thothii-core-sequence.visual-check.1440x900.light.png" + }, + { + "width": 1440, + "height": 900, + "theme": "dark", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1194, + "diagramWidth": 1164, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "dark", + "file": "thothii-core-sequence.visual-check.1440x900.dark.png" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1818, + "diagramWidth": 1768, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light", + "file": "thothii-core-sequence.visual-check.2048x1320.light.png" + }, + { + "width": 2048, + "height": 1320, + "theme": "dark", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1818, + "diagramWidth": 1768, + "viewBoxWidth": 1080, + "minimumProjectedNodeTextPx": 7, + "minimumProjectedNodeText": "browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "dark", + "file": "thothii-core-sequence.visual-check.2048x1320.dark.png" + } + ], + "contactSheet": "thothii-core-sequence.visual-check.html" + }, + "sidecars": { + "receipt": "thothii-core-sequence.visual-check.json", + "contactSheet": "thothii-core-sequence.visual-check.html" + } +} diff --git a/docs/architecture/thothii-core.sequence.json b/docs/architecture/thothii-core.sequence.json new file mode 100644 index 00000000..cd02eccd --- /dev/null +++ b/docs/architecture/thothii-core.sequence.json @@ -0,0 +1,230 @@ +{ + "schema_version": 1, + "diagram_type": "sequence", + "meta": { + "title": "Sequenza runtime del core ThothII", + "output": "thothii-core-sequence.html", + "quality_profile": "showcase", + "column_fit": "spread", + "viewBox": [1080, 540] + }, + "participants": [ + { + "id": "reviewer", + "type": "external", + "label": "Revisore", + "sublabel": "browser" + }, + { + "id": "frontend", + "type": "frontend", + "label": "Frontend", + "sublabel": "React + SSE" + }, + { + "id": "api", + "type": "backend", + "label": "Core API", + "sublabel": "Fastify + SseHub" + }, + { + "id": "runtime", + "type": "backend", + "label": "Runtime", + "sublabel": "manager + bridge" + }, + { + "id": "pi", + "type": "backend", + "label": "Pi + tht-gate", + "sublabel": "RPC per sessione" + }, + { + "id": "model", + "type": "external", + "label": "Modello AI", + "sublabel": "provider configurato" + }, + { + "id": "tht", + "type": "backend", + "label": "tht CLI", + "sublabel": "workflow persistito" + }, + { + "id": "state", + "type": "database", + "label": "Stato e dati", + "sublabel": "workspace + dati" + } + ], + "segments": [ + { + "from": 150, + "to": 300, + "label": "Avvio o ripresa" + }, + { + "from": 305, + "to": 395, + "label": "Turno Pi e workflow persistito" + }, + { + "from": 400, + "to": 475, + "label": "Evento SSE e gate umano" + } + ], + "messages": [ + { + "id": "askQuestion", + "from": "reviewer", + "to": "frontend", + "y": 160, + "label": "domanda o decisione gate", + "variant": "emphasis" + }, + { + "id": "createSession", + "from": "frontend", + "to": "api", + "y": 185, + "label": "POST /sessions o /response", + "variant": "emphasis" + }, + { + "id": "newManifest", + "from": "api", + "to": "tht", + "y": 210, + "label": "sessionNew o sessionShow/reopen" + }, + { + "id": "persistQuestion", + "from": "tht", + "to": "state", + "y": 235, + "label": "manifest, artifact e ledger" + }, + { + "id": "createRuntime", + "from": "api", + "to": "runtime", + "y": 260, + "label": "createFor + bind; configure + searchPack", + "note": "configure(...) e searchPack(...) terminano in parallelo prima dell’avvio del modello." + }, + { + "id": "startPi", + "from": "runtime", + "to": "pi", + "y": 285, + "label": "/nuova-domanda o /riprendi-sessione", + "variant": "emphasis", + "note": "In ripresa usa /riprendi-sessione dopo aver riletto provider, modello e thinking dal manifest." + }, + { + "id": "modelRequest", + "from": "pi", + "to": "model", + "y": 315, + "label": "turno modello; token e tool call" + }, + { + "id": "phaseCommand", + "from": "pi", + "to": "tht", + "y": 345, + "label": "comandi phase / session / evidence", + "variant": "emphasis" + }, + { + "id": "readWriteTruth", + "from": "tht", + "to": "state", + "y": 375, + "label": "legge/scrive artifact e ledger", + "note": "Il workspace persistito è la verità; DWH è read-only e Qdrant serve il retrieval." + }, + { + "id": "rpcEvents", + "from": "pi", + "to": "runtime", + "y": 405, + "label": "text_delta / ui_request" + }, + { + "id": "sseEvent", + "from": "runtime", + "to": "frontend", + "y": 430, + "label": "SSE via SseHub: text · info · widget", + "variant": "return" + }, + { + "id": "showResult", + "from": "frontend", + "to": "reviewer", + "y": 455, + "label": "testo o widget di revisione", + "variant": "return" + } + ], + "activations": [ + { + "participant": "frontend", + "from": 155, + "to": 460, + "type": "frontend" + }, + { + "participant": "api", + "from": 180, + "to": 290, + "type": "backend" + }, + { + "participant": "runtime", + "from": 255, + "to": 440, + "type": "backend" + }, + { + "participant": "pi", + "from": 280, + "to": 415, + "type": "backend" + }, + { + "participant": "model", + "from": 310, + "to": 335, + "type": "external" + }, + { + "participant": "tht", + "from": 205, + "to": 390, + "type": "backend" + }, + { + "participant": "state", + "from": 230, + "to": 390, + "type": "database" + } + ], + "cards": [ + { + "dot": "cyan", + "title": "Bootstrap", + "items": [ + "La sessione viene persistita prima dell’avvio di Pi; configurazione e retrieval terminano prima del primo prompt.", + "La ripresa rifiuta sessioni finalizzate o archiviate e usa sempre /riprendi-sessione .", + "Il modello propone; il revisore decide tramite reviewer_select, reviewer_decide o reviewer_confirm.", + "La risposta percorre POST /sessions/:id/response → bridge.respond → extension_ui_response; stale/duplicate ricevono 409.", + "La fase avanza solo dopo append della decisione esplicita e fold del ledger persistito." + ] + } + ] +} diff --git a/docs/architecture/thothii-runtime.architecture.json b/docs/architecture/thothii-runtime.architecture.json new file mode 100644 index 00000000..00dc400d --- /dev/null +++ b/docs/architecture/thothii-runtime.architecture.json @@ -0,0 +1,267 @@ +{ + "schema_version": 1, + "diagram_type": "architecture", + "meta": { + "title": "ThothII Runtime Architecture", + "locale": "en", + "output": "thothii-runtime.html", + "quality_profile": "showcase", + "viewBox": [1360, 800] + }, + "components": [ + { + "id": "reviewer", + "type": "external", + "label": "User / Reviewer", + "sublabel": "Browser", + "pos": [20, 300], + "size": [145, 72] + }, + { + "id": "frontend", + "type": "frontend", + "label": "Frontend", + "sublabel": "React 18 + Vite", + "tag": "in-memory transcript", + "pos": [205, 295], + "size": [165, 82] + }, + { + "id": "backend", + "type": "backend", + "label": "Backend", + "sublabel": "Fastify + TypeScript", + "tag": "session bridge", + "pos": [410, 295], + "size": [175, 82] + }, + { + "id": "pi", + "type": "backend", + "label": "Pi", + "sublabel": "RPC process per session", + "tag": "inside core", + "pos": [625, 295], + "size": [170, 82] + }, + { + "id": "harness", + "type": "backend", + "label": "tht / Harness", + "sublabel": "8-phase NL→SQL workflow", + "tag": "workflow owner", + "pos": [835, 295], + "size": [190, 82] + }, + { + "id": "workspace", + "type": "database", + "label": "Workspace Repository", + "sublabel": "sessions + phase artifacts", + "tag": "source of truth", + "pos": [1065, 295], + "size": [205, 82] + }, + { + "id": "auth", + "type": "security", + "label": "Authentication", + "sublabel": "local registry or OIDC", + "pos": [410, 70], + "size": [170, 76] + }, + { + "id": "modelProvider", + "type": "external", + "label": "AI Model Provider", + "sublabel": "installation-selected APIs", + "pos": [615, 70], + "size": [190, 76] + }, + { + "id": "catalog", + "type": "database", + "label": "Metadata Catalog", + "sublabel": "PostgreSQL + Kysely", + "tag": "backend-owned", + "pos": [410, 545], + "size": [180, 82] + }, + { + "id": "embedding", + "type": "backend", + "label": "Embedding Service", + "sublabel": "Ollama", + "pos": [800, 545], + "size": [170, 76] + }, + { + "id": "qdrant", + "type": "database", + "label": "Vector Store", + "sublabel": "Qdrant", + "pos": [1010, 545], + "size": [160, 76] + }, + { + "id": "dwh", + "type": "external", + "label": "Data Warehouse", + "sublabel": "external, read-only", + "pos": [1070, 70], + "size": [180, 82] + } + ], + "boundaries": [ + { + "kind": "region", + "label": "Local Docker Compose runtime", + "wraps": ["frontend", "backend", "pi", "harness", "catalog", "embedding", "qdrant"], + "pad": 26 + }, + { + "kind": "security-group", + "label": "core container", + "wraps": ["pi", "harness"], + "pad": 18 + }, + { + "kind": "security-group", + "label": "operator-controlled persisted state", + "wraps": ["workspace"], + "pad": 18 + } + ], + "connections": [ + { + "id": "reviewerInput", + "from": "reviewer", + "to": "frontend", + "label": "question / gate decision", + "variant": "emphasis", + "labelAt": [185, 391] + }, + { + "id": "restRequests", + "from": "frontend", + "to": "backend", + "label": "REST", + "variant": "emphasis" + }, + { + "id": "sseEvents", + "from": "backend", + "to": "frontend", + "label": "SSE: text · info · widget", + "variant": "dashed", + "labelAt": [390, 415] + }, + { + "id": "piRpc", + "from": "backend", + "to": "pi", + "label": "JSON-RPC stdio", + "variant": "emphasis", + "labelAt": [605, 391] + }, + { + "id": "gateCommands", + "from": "pi", + "to": "harness", + "label": "phase / gate commands", + "variant": "emphasis", + "labelAt": [815, 391] + }, + { + "id": "persistArtifacts", + "from": "harness", + "to": "workspace", + "label": "artifacts + decision ledger", + "variant": "emphasis", + "labelAt": [1045, 391] + }, + { + "id": "authenticate", + "from": "auth", + "to": "backend", + "label": "local session / OIDC callback", + "variant": "security", + "fromSide": "bottom", + "toSide": "top" + }, + { + "id": "sessionModel", + "from": "pi", + "to": "modelProvider", + "label": "session model API", + "fromSide": "top", + "toSide": "bottom", + "labelAt": [755, 185] + }, + { + "id": "catalogState", + "from": "backend", + "to": "catalog", + "label": "catalog CRUD + runs", + "fromSide": "bottom", + "toSide": "top" + }, + { + "id": "metadataModel", + "from": "backend", + "to": "modelProvider", + "label": "short-lived LiteLLM calls", + "variant": "dashed" + }, + { + "id": "readOnlySql", + "from": "harness", + "to": "dwh", + "label": "introspection + read-only SQL" + }, + { + "id": "embedRequests", + "from": "harness", + "to": "embedding", + "label": "embedding requests", + "fromSide": "bottom", + "toSide": "top", + "labelAt": [830, 490] + }, + { + "id": "vectorAccess", + "from": "harness", + "to": "qdrant", + "label": "index + vector retrieval", + "fromSide": "bottom", + "toSide": "top", + "labelAt": [1165, 475] + } + ], + "cards": [ + { + "dot": "cyan", + "title": "Primary session path", + "items": [ + "The browser sends questions and reviewer decisions over REST; the backend returns text, information, and widget descriptors over SSE.", + "One Pi RPC process drives each active session and delegates deterministic phase operations to tht / Harness." + ] + }, + { + "dot": "emerald", + "title": "State ownership", + "items": [ + "Harness owns the eight-phase workflow and persists session_manifest.yaml, phase artifacts, and review_decisions.jsonl.", + "Backend owns the separate installation-local metadata catalog; the live frontend transcript is not persisted." + ] + }, + { + "dot": "rose", + "title": "Trust boundaries", + "items": [ + "Catalog DB, Qdrant, and Ollama stay inside Compose; workspace state is operator controlled and mounted into core.", + "The DWH and AI provider remain external endpoints; DWH access is constrained to introspection, tests, bounded samples, and read-only SQL." + ] + } + ] +} diff --git a/docs/architecture/thothii-runtime.html b/docs/architecture/thothii-runtime.html new file mode 100644 index 00000000..25850491 --- /dev/null +++ b/docs/architecture/thothii-runtime.html @@ -0,0 +1,14922 @@ + + + + + + + ThothII Runtime Architecture Diagram + + + + + + + + + + + +
+ +
+
+
+

ThothII Runtime Architecture

+
+
+ + + + + + + +
+ + ThothII Runtime Architecture + An architecture diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + User / Reviewer · Browser · Architecture component + + + + User / Reviewer + Browser + + + + Frontend · React 18 + Vite · Local Docker Compose runtime · in-memory transcript + + + + Frontend + React 18 + Vite + in-memory transcript + + + + Backend · Fastify + TypeScript · Local Docker Compose runtime · session bridge + + + + Backend + Fastify + TypeScript + session bridge + + + + Pi · RPC process per session · Local Docker Compose runtime › core container · inside core + + + + Pi + RPC process per session + inside core + + + + tht / Harness · 8-phase NL→SQL workflow · Local Docker Compose runtime › core container · workflow owner + + + + tht / Harness + 8-phase NL→SQL workflow + workflow owner + + + + Workspace Repository · sessions + phase artifacts · operator-controlled persisted state · source of truth + + + + Workspace Repository + sessions + phase artifacts + source of truth + + + + Authentication · local registry or OIDC · Architecture component + + + + Authentication + local registry or OIDC + + + + AI Model Provider · installation-selected APIs · Architecture component + + + + AI Model Provider + installation-selected APIs + + + + Metadata Catalog · PostgreSQL + Kysely · Local Docker Compose runtime · backend-owned + + + + Metadata Catalog + PostgreSQL + Kysely + backend-owned + + + + Embedding Service · Ollama · Local Docker Compose runtime + + + + Embedding Service + Ollama + + + + Vector Store · Qdrant · Local Docker Compose runtime + + + + Vector Store + Qdrant + + + + Data Warehouse · external, read-only · Architecture component + + + + Data Warehouse + external, read-only + + + + + + question / gate decision + + + + REST + + + + SSE: text · info · widget + + + + JSON-RPC stdio + + + + phase / gate commands + + + + artifacts + decision ledger + + + + local session / OIDC callback + + + + session model API + + + + catalog CRUD + runs + + + + short-lived LiteLLM calls + + + + introspection + read-only SQL + + + + embedding requests + + + + index + vector retrieval + + + + + + Local Docker Compose runtime + + + + + core container + + + + + operator-controlled persisted state + + + + + Legend + + + Frontend + + + + Backend + + + + Database + + + + Security + + + + External + + + +

+ + + + + + + + + +
+ + +
+
+
+
+

Primary session path

+
+
    +
  • • The browser sends questions and reviewer decisions over REST; the backend returns text, information, and widget descriptors over SSE.
    • +
  • • One Pi RPC process drives each active session and delegates deterministic phase operations to tht / Harness.
    • +
+
+ +
+
+
+

State ownership

+
+
    +
  • • Harness owns the eight-phase workflow and persists session_manifest.yaml, phase artifacts, and review_decisions.jsonl.
    • +
  • • Backend owns the separate installation-local metadata catalog; the live frontend transcript is not persisted.
    • +
+
+ +
+
+
+

Trust boundaries

+
+
    +
  • • Catalog DB, Qdrant, and Ollama stay inside Compose; workspace state is operator controlled and mounted into core.
    • +
  • • The DWH and AI provider remain external endpoints; DWH access is constrained to introspection, tests, bounded samples, and read-only SQL.
    • +
+
+
+ +
+ + + + diff --git a/docs/architecture/thothii-runtime.visual-check.1440x900.dark.png b/docs/architecture/thothii-runtime.visual-check.1440x900.dark.png new file mode 100644 index 00000000..b9933d60 Binary files /dev/null and b/docs/architecture/thothii-runtime.visual-check.1440x900.dark.png differ diff --git a/docs/architecture/thothii-runtime.visual-check.1440x900.light.png b/docs/architecture/thothii-runtime.visual-check.1440x900.light.png new file mode 100644 index 00000000..074e1a8b Binary files /dev/null and b/docs/architecture/thothii-runtime.visual-check.1440x900.light.png differ diff --git a/docs/architecture/thothii-runtime.visual-check.2048x1320.dark.png b/docs/architecture/thothii-runtime.visual-check.2048x1320.dark.png new file mode 100644 index 00000000..105395a9 Binary files /dev/null and b/docs/architecture/thothii-runtime.visual-check.2048x1320.dark.png differ diff --git a/docs/architecture/thothii-runtime.visual-check.2048x1320.light.png b/docs/architecture/thothii-runtime.visual-check.2048x1320.light.png new file mode 100644 index 00000000..9a94abd3 Binary files /dev/null and b/docs/architecture/thothii-runtime.visual-check.2048x1320.light.png differ diff --git a/docs/architecture/thothii-runtime.visual-check.html b/docs/architecture/thothii-runtime.visual-check.html new file mode 100644 index 00000000..a6db65f1 --- /dev/null +++ b/docs/architecture/thothii-runtime.visual-check.html @@ -0,0 +1,32 @@ + + + + + +Archify automated browser evidence · thothii-runtime.html + + + +

Automated browser evidence

thothii-runtime.html · visual-check containment pass · perceptual visual review pending

+
+
+ light 1440 by 900 +
LIGHT · 1440×900 · containment pass
+
+
+ dark 1440 by 900 +
DARK · 1440×900 · containment pass
+
+
+ light 2048 by 1320 +
LIGHT · 2048×1320 · containment pass
+
+
+ dark 2048 by 1320 +
DARK · 2048×1320 · containment pass
+
+
+ + diff --git a/docs/architecture/thothii-runtime.visual-check.json b/docs/architecture/thothii-runtime.visual-check.json new file mode 100644 index 00000000..7d2356e0 --- /dev/null +++ b/docs/architecture/thothii-runtime.visual-check.json @@ -0,0 +1,548 @@ +{ + "schemaVersion": 1, + "ok": true, + "command": "visual-check", + "evidenceKind": "automated-browser", + "status": "pass", + "visualReview": "pending", + "artifact": { + "path": "/Users/mp/projects/ThothII/docs/architecture/thothii-runtime.html", + "sha256": "ad19195852c7c643228663e5bf7f3cb273afb0456fedbe295d4df24bc345b6c7", + "bytes": 724277 + }, + "state": { + "detail": "read", + "motion": "still" + }, + "chrome": { + "status": "available", + "executable": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" + }, + "diagnostics": [], + "containment": { + "status": "pass", + "viewports": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 968, + "diagramWidth": 938, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.20735294117647, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1600, + "height": 1000, + "theme": "light", + "innerWidth": 1600, + "innerHeight": 1000, + "scrollWidth": 1600, + "scrollHeight": 1000, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1009, + "diagramWidth": 979, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.478676470588235, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1920, + "height": 1080, + "theme": "light", + "innerWidth": 1920, + "innerHeight": 1080, + "scrollWidth": 1920, + "scrollHeight": 1080, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1172, + "diagramWidth": 1142, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 7.557352941176471, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1583, + "diagramWidth": 1533, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 9, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + } + ] + }, + "readability": { + "status": "pass", + "minimumProjectedNodeTextPx": 6, + "viewports": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 968, + "diagramWidth": 938, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.20735294117647, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1600, + "height": 1000, + "theme": "light", + "innerWidth": 1600, + "innerHeight": 1000, + "scrollWidth": 1600, + "scrollHeight": 1000, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1009, + "diagramWidth": 979, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.478676470588235, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1920, + "height": 1080, + "theme": "light", + "innerWidth": 1920, + "innerHeight": 1080, + "scrollWidth": 1920, + "scrollHeight": 1080, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1172, + "diagramWidth": 1142, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 7.557352941176471, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1583, + "diagramWidth": 1533, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 9, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + } + ] + }, + "viewerChrome": { + "status": "pass", + "viewports": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 968, + "diagramWidth": 938, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.20735294117647, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1600, + "height": 1000, + "theme": "light", + "innerWidth": 1600, + "innerHeight": 1000, + "scrollWidth": 1600, + "scrollHeight": 1000, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1009, + "diagramWidth": 979, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.478676470588235, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 1920, + "height": 1080, + "theme": "light", + "innerWidth": 1920, + "innerHeight": 1080, + "scrollWidth": 1920, + "scrollHeight": 1080, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1172, + "diagramWidth": 1142, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 7.557352941176471, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1583, + "diagramWidth": 1533, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 9, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light" + } + ] + }, + "captures": { + "status": "pass", + "screenshots": [ + { + "width": 1440, + "height": 900, + "theme": "light", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 968, + "diagramWidth": 938, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.20735294117647, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light", + "file": "thothii-runtime.visual-check.1440x900.light.png" + }, + { + "width": 1440, + "height": 900, + "theme": "dark", + "innerWidth": 1440, + "innerHeight": 900, + "scrollWidth": 1440, + "scrollHeight": 900, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 968, + "diagramWidth": 938, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 6.20735294117647, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 51, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "dark", + "file": "thothii-runtime.visual-check.1440x900.dark.png" + }, + { + "width": 2048, + "height": 1320, + "theme": "light", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1583, + "diagramWidth": 1533, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 9, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "light", + "file": "thothii-runtime.visual-check.2048x1320.light.png" + }, + { + "width": 2048, + "height": 1320, + "theme": "dark", + "innerWidth": 2048, + "innerHeight": 1320, + "scrollWidth": 2048, + "scrollHeight": 1320, + "overflowX": false, + "overflowY": false, + "ok": true, + "readerWidth": 1583, + "diagramWidth": 1533, + "viewBoxWidth": 1360, + "minimumProjectedNodeTextPx": 9, + "minimumProjectedNodeText": "Browser", + "minimumProjectedNodeTextDetail": "context", + "minimumRequiredNodeTextPx": 6, + "readabilityOk": true, + "hasLegend": true, + "hasNavigationDock": true, + "legendDockIntersectionArea": 0, + "dockStageIntersectionArea": 0, + "dockStageGap": 10.21875, + "requiredDockStageGap": 10, + "viewerChromeStageOk": true, + "viewerChromeReserve": 41, + "viewerChromeActive": true, + "viewerChromeOk": true, + "resolvedTheme": "dark", + "file": "thothii-runtime.visual-check.2048x1320.dark.png" + } + ], + "contactSheet": "thothii-runtime.visual-check.html" + }, + "sidecars": { + "receipt": "thothii-runtime.visual-check.json", + "contactSheet": "thothii-runtime.visual-check.html" + } +} diff --git a/docs/operations/database-management.md b/docs/operations/database-management.md index d7481f23..ac45bf49 100644 --- a/docs/operations/database-management.md +++ b/docs/operations/database-management.md @@ -60,7 +60,8 @@ remains available only until the integrated Fleet Ledger surface passes owner ac complete the binding fields that the chosen transport requires. 3. Enter secrets only when replacing them. They remain write-only and are never returned by the application. -4. Run **Test connection** before any synchronization. +4. Use **Test connection** whenever you want an informational connectivity check. Its result does + not enable or disable catalog operations. SSH uses a private key, optional key passphrase, mandatory `known_hosts`, and optional PostgreSQL TLS CA/server name. REST prefers `POST /rpc/schema_snapshot`; when it is absent, the catalog may @@ -71,7 +72,9 @@ capability, malformed snapshot, or connector error applies no catalog changes. S ## Synchronize authoritative schema metadata Schema synchronization reads the external database and reconciles the installation-local catalog. -It never changes the source database. The available synchronization scopes are **tables**, +It never changes the source database. Every synchronization attempts a fresh connection when it +runs; an unreachable server or rejected credential fails that run without changing catalog data. +The available synchronization scopes are **tables**, **columns**, **relationships**, and **all**, but the UI exposes them at different levels: | Location | Action | Effective scope | @@ -88,10 +91,10 @@ The selection requirement in the tables and columns views controls whether the a be used; it is not always the same as the synchronization target. The **Sync all** button in the tables view is the direct shortcut for the full-database scope. -Before starting any synchronization, run **Test connection**. Synchronization is rejected when -the binding is unreachable or its tested version is older than the current binding configuration. -Only one catalog operation can be active for a database at a time; explicit cleanup shares this -exclusion. +Connection tests are informational and are never a synchronization prerequisite. Each +synchronization tests its own access while reading the schema; an unavailable connection fails +that operation with a connector error. Only one catalog operation can be active for a database at +a time; explicit cleanup shares this exclusion. ### What a synchronization does @@ -167,6 +170,12 @@ starting generation. Changing a flag affects future generations only; existing g descriptions are not regenerated. Real and substituted samples remain transient and are not persisted or returned to the browser. +The database-level **Copy generated descriptions to all columns** action applies every non-empty +AI-generated column description to the corresponding curated **Description** field in one atomic +operation. It skips empty generated descriptions, reports copied and skipped counts, and retains the +generated text. Because this can replace reviewed descriptions, the interface requires explicit +confirmation before applying it. + The decisions behind this surface are [ADRs 0001–0011](../adr/0001-postgres-metadata-catalog.md) and the detailed acceptance record is [AI catalog description generation acceptance](../testing/2026-08-29-ai-catalog-description-generation-acceptance.md). diff --git a/docs/operations/sensitivity-analysis.md b/docs/operations/sensitivity-analysis.md index ebf7f295..ef027e04 100644 --- a/docs/operations/sensitivity-analysis.md +++ b/docs/operations/sensitivity-analysis.md @@ -7,9 +7,13 @@ may set either value, including overriding a `sensitive` proposal. ## Default policy -`SensitivityClassifier` is the only column-level decision point. The versioned `sensitivity-v2` +`SensitivityClassifier` is the only column-level decision point. The versioned `sensitivity-v4` policy combines: +- a structural exclusion for declared `bigint` primary-key columns and undeclared `bigint` + columns following the exact `pk` naming convention; their values are non-informative identifiers + and are therefore not inspected as possible sensitive content. The evidence distinguishes + declared constraints from convention-based inference; - normalized column-name rules for direct identifiers, credentials, and health data; - validated content rules for email, Italian fiscal code and VAT, passport, identity-card and driving-licence identifiers, phone numbers, IBAN/BIC, payment-card checksums, IP/MAC addresses, @@ -46,7 +50,7 @@ and returns no review instead of manufacturing `unknown` decisions. There is no global sixty-second analysis deadline. Work is bounded by sample counts, per-query timeouts, and early column exits. The operation is interrupted only when its request connection is aborted or the backend restarts. Historical or interrupted run counters named `unknown` represent -columns that were not processed; `unknown` is not a `sensitivity-v2` column assessment. +columns that were not processed; `unknown` is not a `sensitivity-v4` column assessment. History stores only the policy version, aggregate outcomes, timestamps, and fixed operational events. Sanitized rule IDs are returned in the transient review and shadow report, not persisted. diff --git a/docs/operations/workspaces.md b/docs/operations/workspaces.md index 40840002..88c44a31 100644 --- a/docs/operations/workspaces.md +++ b/docs/operations/workspaces.md @@ -36,8 +36,10 @@ the curator-owned repository during pull. keys, or signed URLs in this repository. 2. In the application, update the workspace repository. This fetches and validates the candidate; it never edits the remote repository. -3. Select the workspace. Supply or replace its write-only runtime secrets, then run **Validate - workspace source** and **Test workspace connections**. +3. Select the workspace. Supply or replace its write-only Evidence runtime secrets, configure its + database in **Database Management**, then run **Validate workspace source** and **Test workspace + connections**. The workspace connection test uses that same current database configuration for + DWH connectivity and also checks the workspace Evidence and installation semantic services. 4. Select it as the installation workspace before creating sessions. 5. Use the host CLI for preprocessing. It dispatches a profile-gated maintenance service and returns a single structured result; `--json` keeps stdout machine-readable. @@ -71,8 +73,9 @@ forms and the schema-v4 descriptor contract, see ## Transport and revision rules Runtime sessions support direct PostgreSQL and REST bindings. SSH tunnel bindings are diagnostic -only for this path, so they cannot admit an NL→SQL session. Database Management has its own -strict known-host SSH path for connection tests and schema synchronization. +only for this path, so they cannot admit an NL→SQL session. Database Management and **Test +workspace connections** share the current database binding, including its strict known-host SSH +path; the remaining workspace diagnostics cover Evidence and installation semantic services. Every new session pins the active Git revision. Snapshot cleanup retains revisions still referenced by unarchived sessions. A later pull can prepare a future session but cannot alter a diff --git a/frontend/src/api/catalog-databases.ts b/frontend/src/api/catalog-databases.ts index 21da4be8..e1cd72a1 100644 --- a/frontend/src/api/catalog-databases.ts +++ b/frontend/src/api/catalog-databases.ts @@ -68,6 +68,17 @@ export interface CatalogDatabase { secrets: Record; } +export type CatalogDatabaseTestResult = Omit< + CatalogDatabase, + | "workspaceName" + | "workspaceDescription" + | "workspaceAvailable" + | "workspaceRevision" + | "workspaceEvidence" + | "runtimeBinding" + | "activeSyncRun" +>; + export interface CatalogMetrics { scope: "global" | "database"; databaseId: string | null; @@ -119,6 +130,7 @@ export interface CatalogColumn { description: string | null; generatedDescription: string | null; sensitive: boolean; + sensitivityReason?: string | null; lastSyncedDatabaseVersion: number | null; lastSyncedAt: string | null; version: number; @@ -235,7 +247,7 @@ export interface CatalogMetadataDeleteCounts { relationships: number; } -export type CatalogDescriptionTarget = "tables" | "columns"; +export type CatalogDescriptionTarget = "tables" | "columns" | "database_columns"; export interface CatalogDescriptionConsolidationCounts { copied: number; @@ -421,7 +433,7 @@ export const replaceCatalogDatabaseSecrets = ( }); export const testCatalogDatabase = (id: string, version: number) => - apiFetch(`/catalog/databases/${encodeURIComponent(id)}/test`, { + apiFetch(`/catalog/databases/${encodeURIComponent(id)}/test`, { method: "POST", body: JSON.stringify({ version }), }); @@ -465,9 +477,10 @@ export const updateCatalogColumnSensitive = ( columnId: string, version: number, sensitive: boolean, + sensitivityReason?: string | null, ) => apiFetch( `/catalog/databases/${encodeURIComponent(databaseId)}/tables/${encodeURIComponent(tableId)}/columns/${encodeURIComponent(columnId)}`, - { method: "PATCH", body: JSON.stringify({ version, sensitive }) }, + { method: "PATCH", body: JSON.stringify({ version, sensitive, sensitivityReason }) }, ); export const runSensitivityAnalysis = ( @@ -544,14 +557,28 @@ export const deleteCatalogTableMetadata = ( { method: "POST", body: JSON.stringify({ tableIds, target }) }, ); -export const consolidateCatalogDescriptions = ( +export function consolidateCatalogDescriptions( + databaseId: string, + target: "database_columns", +): Promise; +export function consolidateCatalogDescriptions( + databaseId: string, + target: Exclude, + targetIds: string[], +): Promise; +export function consolidateCatalogDescriptions( databaseId: string, target: CatalogDescriptionTarget, - targetIds: string[], -) => apiFetch( - `/catalog/databases/${encodeURIComponent(databaseId)}/descriptions/consolidate`, - { method: "POST", body: JSON.stringify({ target, targetIds }) }, -); + targetIds?: string[], +): Promise { + return apiFetch( + `/catalog/databases/${encodeURIComponent(databaseId)}/descriptions/consolidate`, + { + method: "POST", + body: JSON.stringify(target === "database_columns" ? { target } : { target, targetIds }), + }, + ); +} export const startCatalogSync = ( databaseId: string, diff --git a/frontend/src/api/client.test.ts b/frontend/src/api/client.test.ts index 2113b3ce..033d2e48 100644 --- a/frontend/src/api/client.test.ts +++ b/frontend/src/api/client.test.ts @@ -153,6 +153,7 @@ test.each([ ["sensitive_data_suggestion_history_request_invalid", "Sensitivity analysis history parameters are invalid."], ["sensitive_data_suggestion_history_failed", "Sensitivity analysis history could not be loaded."], ["sensitive_data_suggestion_run_not_found", "The sensitivity analysis run was not found."], + ["schema_introspection_failed", "The database schema could not be read. Check the connection and credentials, then try again."], ["relationship_not_found", "The relationship no longer exists. Refresh and try again."], ["relationship_duplicate", "This relationship already exists."], ["relationship_target_not_unique", "The target column must be the only primary-key column of its table."], diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 6bd3dc2b..f82b9754 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -101,7 +101,7 @@ const localCodeMessages: Record = { sensitive_data_suggestion_history_failed: "Sensitivity analysis history could not be loaded.", sensitive_data_suggestion_run_not_found: "The sensitivity analysis run was not found.", schema_sync_conflict: "A schema synchronization is already active or no longer current.", - schema_introspection_failed: "The database schema could not be read safely.", + schema_introspection_failed: "The database schema could not be read. Check the connection and credentials, then try again.", schema_request_invalid: "The schema request is invalid.", schema_operation_failed: "The schema operation failed.", sync_run_not_found: "The synchronization run was not found.", diff --git a/frontend/src/api/workspaces.test.ts b/frontend/src/api/workspaces.test.ts index f298a673..659127b2 100644 --- a/frontend/src/api/workspaces.test.ts +++ b/frontend/src/api/workspaces.test.ts @@ -70,17 +70,52 @@ test("accepts the evidence schema version materialized by the backend", async () }))); await expect(getWorkspace("psd-clinical")).resolves.toEqual({ - workspace: { - ...persistedWorkspace, - evidence: { - source: persistedWorkspace.evidence.source, - policy: persistedWorkspace.evidence.policy, - }, - }, + workspace: persistedWorkspace, revision, }); }); +test("validates a workspace read from the backend without losing its Evidence schema version", async () => { + const persistedWorkspace = { + ...workspace, + evidence: { + schema_version: 1, + source: { + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: ["curated/**/*.md"], + max_bytes: 10 * 1024 * 1024, + }, + policy: { max_chunk_chars: 5000, retain_published_generations: 3 }, + }, + }; + let validationRequest: unknown; + server.use( + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: persistedWorkspace, + revision, + })), + http.post("/api/workspaces/validate", async ({ request }) => { + validationRequest = await request.json(); + return HttpResponse.json({ + workspace: persistedWorkspace, + contract: {}, + activatable: true, + diagnostics: [], + authentication: { + ready: true, + mode: "local", + checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }], + }, + }); + }), + ); + + const loaded = await getWorkspace("psd-clinical"); + await expect(validateWorkspace(loaded.workspace)).resolves.toMatchObject({ activatable: true }); + expect(validationRequest).toEqual({ workspace: persistedWorkspace }); +}); + test("decodes runtime requirements but rejects any secret value returned by the server", async () => { server.use(http.get( "/api/workspaces/psd-clinical/runtime-configuration", diff --git a/frontend/src/api/workspaces.ts b/frontend/src/api/workspaces.ts index ee3e88eb..b663d783 100644 --- a/frontend/src/api/workspaces.ts +++ b/frontend/src/api/workspaces.ts @@ -56,6 +56,7 @@ export type EvidenceSource = }; export interface WorkspaceEvidence { + schema_version: 1 | 2; source: EvidenceSource; policy: EvidencePolicy; } diff --git a/frontend/src/shell/DatabaseManagementPage.test.tsx b/frontend/src/shell/DatabaseManagementPage.test.tsx index ef07fb37..4147a268 100644 --- a/frontend/src/shell/DatabaseManagementPage.test.tsx +++ b/frontend/src/shell/DatabaseManagementPage.test.tsx @@ -1,4 +1,4 @@ -import { act, render, screen, waitFor, within } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { http, HttpResponse } from "msw"; @@ -6,6 +6,7 @@ import { server } from "../test/msw"; import type { CatalogColumn, CatalogDatabase, + CatalogRelationship, CatalogSyncRun, CatalogTable, DescriptionGenerationRun, @@ -376,6 +377,84 @@ test("opens the relationship map directly from a Fleet database and restores foc })).toHaveFocus()); }); +test("refreshes the relationship KPI after building generated relationships", async () => { + const user = userEvent.setup(); + const generatedRelationship: CatalogRelationship = { + id: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", + databaseId: "11111111-1111-4111-8111-111111111111", + constraintName: null, + sourceTableId: "ffffffff-ffff-4fff-8fff-ffffffffffff", + sourceTableName: "visits", + targetTableId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + targetTableName: "patients", + updateRule: null, + deleteRule: null, + deferrable: false, + initiallyDeferred: false, + columns: [{ + position: 1, + sourceColumnId: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + sourceColumnName: "patient_id", + targetColumnId: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", + targetColumnName: "id", + }], + origin: "generated", + status: "active", + lastSyncedDatabaseVersion: null, + lastSyncedAt: null, + createdAt: "2026-08-27T10:00:00Z", + updatedAt: "2026-08-27T10:00:00Z", + }; + let relationships: CatalogRelationship[] = []; + server.use( + http.get("/api/catalog/metrics", ({ request }) => HttpResponse.json({ + scope: new URL(request.url).searchParams.has("databaseId") ? "database" : "global", + databaseId: new URL(request.url).searchParams.get("databaseId"), + tables: 2, + columns: 3, + sensitiveColumns: 0, + relationships: relationships.length, + descriptionTargets: 5, + describedTargets: 0, + descriptionCoverage: 0, + updatedAt: "2026-08-27T10:00:00Z", + })), + http.get( + "/api/catalog/databases/:databaseId/relationships", + () => HttpResponse.json(relationships), + ), + http.post( + "/api/catalog/databases/:databaseId/relationships/rebuild-generated", + () => { + relationships = [generatedRelationship]; + return HttpResponse.json({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 }); + }, + ), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + presentation: "fleet", + }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("button", { + name: "View relationships for Policlinico San Donato", + })); + const summary = await screen.findByRole("region", { name: "Database summary" }); + const relationshipMetric = within(summary).getByText("Relationships").nextElementSibling; + expect(relationshipMetric).toHaveTextContent("0"); + + await user.selectOptions( + screen.getByRole("combobox", { name: "Relationship action" }), + "rebuild-generated", + ); + await user.click(screen.getByRole("button", { name: "Run action" })); + + expect(await screen.findByText("Build complete: 1 added, 0 already present, 0 excluded, 0 ambiguous.")) + .toBeVisible(); + await waitFor(() => expect(relationshipMetric).toHaveTextContent("1")); +}); + test("offers catalog configuration directly on an unconfigured Fleet workspace", async () => { const user = userEvent.setup(); renderPage({ rows: [unconfigured], presentation: "fleet" }); @@ -473,7 +552,10 @@ test("reopens database synchronization history when no run is active", async () renderPage({ rows: [makeDatabase()], presentation: "fleet" }); const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); - await user.click(within(databaseRow).getByText("Policlinico San Donato")); + await user.click(within(databaseRow).getByRole("button", { + name: "View Policlinico San Donato", + })); + await user.click(screen.getByRole("button", { name: "Sync history" })); const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" }); expect(drawer).toBeVisible(); @@ -756,6 +838,55 @@ test("keeps Fleet Generate missing descriptions behind the source-data disclosur expect(generationStarts).toBe(0); }); +test("copies generated descriptions to every database column after explicit confirmation", async () => { + const user = userEvent.setup(); + let consolidationBody: unknown; + server.use( + http.post("/api/catalog/databases/:databaseId/descriptions/consolidate", async ({ request }) => { + consolidationBody = await request.json(); + return HttpResponse.json({ copied: 7, skipped: 2 }); + }), + ); + const database = makeDatabase(); + const { client } = renderPage({ rows: [database], presentation: "fleet" }); + const tablesQuery = ["catalog-tables", database.id] as const; + const firstColumnsQuery = ["catalog-columns", database.id, "table-one"] as const; + const secondColumnsQuery = ["catalog-columns", database.id, "table-two"] as const; + const unrelatedColumnsQuery = ["catalog-columns", "other-database", "table-one"] as const; + for (const queryKey of [ + tablesQuery, + firstColumnsQuery, + secondColumnsQuery, + unrelatedColumnsQuery, + ]) client.setQueryData(queryKey, []); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); + await user.selectOptions( + screen.getByRole("combobox", { name: "Batch action" }), + "consolidate-columns", + ); + await user.click(screen.getByRole("button", { name: "Run action" })); + + expect(screen.getByText( + "Copy generated descriptions to all columns in Policlinico San Donato?", + )).toBeVisible(); + expect(screen.getByText(/replace the current Description value/i)).toBeVisible(); + expect(consolidationBody).toBeUndefined(); + + await user.click(screen.getByRole("button", { name: "Copy to all columns" })); + + await waitFor(() => expect(consolidationBody).toEqual({ target: "database_columns" })); + expect(await screen.findByText("Copied 7 column descriptions; skipped 2")).toBeVisible(); + await waitFor(() => { + expect(client.getQueryState(tablesQuery)?.isInvalidated).toBe(true); + expect(client.getQueryState(firstColumnsQuery)?.isInvalidated).toBe(true); + expect(client.getQueryState(secondColumnsQuery)?.isInvalidated).toBe(true); + }); + expect(client.getQueryState(unrelatedColumnsQuery)?.isInvalidated).toBe(false); + expect(screen.getByText("1 selected")).toBeVisible(); +}); + test("confirms generating all descriptions, supports cancel, and sends the database-wide scope", async () => { const user = userEvent.setup(); const queuedRun = makeDescriptionGenerationRun({ scope: "all", total: 6 }); @@ -989,9 +1120,9 @@ test("disables database-wide generation while a description generation is active await user.click(await screen.findByRole("menuitem", { name: "Generate missing descriptions" })); await user.click(screen.getByRole("button", { name: "Generate missing descriptions" })); expect(await screen.findByRole("dialog", { name: "Description generation" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Close description generation" })); databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); - await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); await user.click(screen.getByRole("button", { name: "Actions" })); expect(await screen.findByRole("menuitem", { name: "Generate all descriptions" })) .toHaveAttribute("aria-disabled", "true"); @@ -1248,7 +1379,6 @@ test("uses an in-page destructive form and returns the YAML workspace to Not con expect(deleteVersion).toBe("3"); expect(await screen.findByRole("button", { name: "Configure catalog for Policlinico San Donato" })).toBeEnabled(); expect(screen.queryByRole("button", { name: "Delete Policlinico San Donato" })).not.toBeInTheDocument(); - expect(screen.getByText("Not configured")).toBeVisible(); }); test("tests only the persisted version and updates the visible connection status", async () => { @@ -1258,7 +1388,9 @@ test("tests only the persisted version and updates the visible connection status server.use(http.post("/api/catalog/databases/:id/test", async ({ request }) => { testBody = await request.json(); row = makeDatabase({ version: 4, connectionStatus: "reachable", testedVersion: 4 }); - return HttpResponse.json(row); + const testResponse: Partial = { ...row }; + delete testResponse.workspaceName; + return HttpResponse.json(testResponse); })); renderPage({ rows: () => [row] }); @@ -1266,9 +1398,78 @@ test("tests only the persisted version and updates the visible connection status await user.click(screen.getByRole("button", { name: "Test connection" })); await waitFor(() => expect(testBody).toEqual({ version: 3 })); + const resultDialog = await screen.findByRole("dialog", { name: "Connection test successful" }); + expect(resultDialog).toBeVisible(); + expect(within(resultDialog).getByText("Policlinico San Donato is reachable.")).toBeVisible(); + await user.click(within(resultDialog).getByRole("button", { name: "Done" })); expect(within(screen.getByRole("region", { name: "Database details form" })).getByText("Reachable")).toBeVisible(); }); +test("reports an unsuccessful selected connection test as a failure", async () => { + const user = userEvent.setup(); + const failed = makeDatabase({ + connectionStatus: "failed", + testedVersion: 3, + lastErrorMessage: "Password authentication failed", + }); + const testResponse: Partial = { ...failed }; + delete testResponse.workspaceName; + server.use(http.post("/api/catalog/databases/:id/test", () => HttpResponse.json(testResponse))); + renderPage({ rows: [makeDatabase()] }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); + await user.click(screen.getByRole("button", { name: "Actions" })); + await user.click(await screen.findByRole("menuitem", { name: "Test connections" })); + + const resultDialog = await screen.findByRole("dialog", { name: "Connection test failed" }); + expect(resultDialog).toBeVisible(); + expect(within(resultDialog).getByText(/Policlinico San Donato: Password authentication failed/)).toBeVisible(); + expect(screen.queryByText("1 connection tested")).not.toBeInTheDocument(); +}); + +test("keeps Fleet synchronization available after an informative connection failure", async () => { + const user = userEvent.setup(); + renderPage({ + rows: [makeDatabase({ + connectionStatus: "failed", + testedVersion: 3, + lastErrorCode: "connector_unavailable", + lastErrorMessage: "Password authentication failed", + })], + presentation: "fleet", + }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); + const actionPicker = screen.getByRole("combobox", { name: "Batch action" }); + await user.selectOptions(actionPicker, "sync-all"); + + expect(within(actionPicker).getByRole("option", { name: "Synchronize all" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Run action" })).toBeEnabled(); +}); + +test("reports a successful selected connection test as a success", async () => { + const user = userEvent.setup(); + const reachable = makeDatabase({ + connectionStatus: "reachable", + testedVersion: 3, + }); + const testResponse: Partial = { ...reachable }; + delete testResponse.workspaceName; + server.use(http.post("/api/catalog/databases/:id/test", () => HttpResponse.json(testResponse))); + renderPage({ rows: [makeDatabase()] }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); + await user.click(screen.getByRole("button", { name: "Actions" })); + await user.click(await screen.findByRole("menuitem", { name: "Test connections" })); + + const resultDialog = await screen.findByRole("dialog", { name: "Connection test successful" }); + expect(resultDialog).toBeVisible(); + expect(within(resultDialog).getByText("Policlinico San Donato is reachable.")).toBeVisible(); +}); + test("retains a stale draft and requires an explicit reload", async () => { const user = userEvent.setup(); server.use( @@ -1442,6 +1643,147 @@ const patientIdColumn: CatalogColumn = { updatedAt: "2026-08-27T10:00:00Z", }; +test("keeps column synchronization available when the latest connection test failed", async () => { + const user = userEvent.setup(); + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.get("/api/catalog/databases/:databaseId/tables/:tableId/columns", () => HttpResponse.json([patientIdColumn])), + ); + renderPage({ + rows: [makeDatabase({ + connectionStatus: "failed", + testedVersion: 3, + lastErrorCode: "connector_unavailable", + lastErrorMessage: "The database connector could not be reached or authenticated.", + })], + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + await user.click(await screen.findByRole("button", { name: "View columns for patients" })); + const rows = await screen.findAllByRole("row", { name: /Patient identifier/ }); + const selectableRow = rows.find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i })); + expect(selectableRow).toBeDefined(); + await user.click(within(selectableRow!).getByRole("checkbox", { name: /toggle row selection/i })); + await user.click(screen.getByRole("button", { name: "Actions" })); + + expect(await screen.findByRole("menuitem", { name: "Synchronize columns for this table" })) + .not.toHaveAttribute("aria-disabled", "true"); +}); + +test("saves a manual sensitive flag change from the Fleet columns toolbar", async () => { + const user = userEvent.setup(); + const nameColumn: CatalogColumn = { + ...patientIdColumn, + id: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", + name: "name", + ordinalPosition: 2, + primaryKeyPosition: null, + isPrimaryKey: false, + sourceComment: "Patient name", + }; + const columns = [patientIdColumn, nameColumn]; + const patches: Array<{ columnId: string; body: Record }> = []; + let requestInFlight = false; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.get( + "/api/catalog/databases/:databaseId/tables/:tableId/columns", + () => HttpResponse.json(columns), + ), + http.patch( + "/api/catalog/databases/:databaseId/tables/:tableId/columns/:columnId", + async ({ params, request }) => { + if (requestInFlight) { + return HttpResponse.json({ + code: "catalog_operation_active", + message: "Another catalog operation is in progress.", + }, { status: 409 }); + } + requestInFlight = true; + const body = await request.json() as Record; + await new Promise((resolve) => window.setTimeout(resolve, 20)); + patches.push({ columnId: String(params.columnId), body }); + requestInFlight = false; + const current = columns.find((column) => column.id === params.columnId)!; + return HttpResponse.json({ ...current, ...body, version: current.version + 1 }); + }, + ), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + presentation: "fleet", + }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("button", { + name: "View tables for Policlinico San Donato", + })); + await user.click(await screen.findByRole("button", { name: "View columns for patients" })); + + expect(screen.queryByRole("button", { name: "Save sensitive fields" })).not.toBeInTheDocument(); + await user.click(await screen.findByRole("checkbox", { name: "Sensitive data for id" })); + await user.click(screen.getByRole("checkbox", { name: "Sensitive data for name" })); + + const saveButton = screen.getByRole("button", { name: "Save sensitive fields" }); + expect(saveButton).toBeVisible(); + await user.click(saveButton); + + await waitFor(() => expect(patches).toHaveLength(2)); + expect(patches).toEqual(expect.arrayContaining([ + expect.objectContaining({ + columnId: patientIdColumn.id, + body: expect.objectContaining({ version: patientIdColumn.version, sensitive: true }), + }), + expect.objectContaining({ + columnId: nameColumn.id, + body: expect.objectContaining({ version: nameColumn.version, sensitive: true }), + }), + ])); + expect(await screen.findByText("Sensitive fields saved")).toBeVisible(); + await waitFor(() => { + expect(screen.queryByRole("button", { name: "Save sensitive fields" })).not.toBeInTheDocument(); + }); +}); + +test("shows a read-only sensitivity reason in the Fleet column metadata matrix", async () => { + const user = userEvent.setup(); + const sensitiveColumn: CatalogColumn = { + ...patientIdColumn, + sensitive: true, + sensitivityReason: "Local assessment matched content rule pii.email.", + }; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.get( + "/api/catalog/databases/:databaseId/tables/:tableId/columns", + () => HttpResponse.json([sensitiveColumn]), + ), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + presentation: "fleet", + }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("button", { + name: "View tables for Policlinico San Donato", + })); + await user.click(await screen.findByRole("button", { name: "View columns for patients" })); + await user.click(await screen.findByRole("button", { name: "Edit metadata for id" })); + + const drawer = await screen.findByRole("dialog", { name: "Review column description" }); + const matrix = within(drawer).getByRole("group", { name: "Column metadata fields" }); + expect(matrix).toHaveClass("sm:grid-cols-2"); + expect(within(matrix).getByLabelText("Source comment")).toHaveAttribute("readonly"); + expect(within(matrix).getByLabelText("Description")).not.toHaveAttribute("readonly"); + expect(within(matrix).getByLabelText("Generated description")).not.toHaveAttribute("readonly"); + expect(within(matrix).getByLabelText("Sensitive motivation")).toHaveValue( + "Local assessment matched content rule pii.email.", + ); + expect(within(matrix).getByLabelText("Sensitive motivation")).toHaveAttribute("readonly"); +}); + test("filters catalog tables as the operator types", async () => { const user = userEvent.setup(); const mediciTable: CatalogTable = { @@ -2050,6 +2392,98 @@ test("shows database sensitivity analysis only for a selection and rejects multi expect(suggestionCalls).toBe(0); }); +test("opens sensitivity analysis progress while the assessment request is still running", async () => { + const user = userEvent.setup(); + let analysisStarted = false; + let releaseAnalysis!: () => void; + const analysisGate = new Promise((resolve) => { + releaseAnalysis = resolve; + }); + const runningRun = makeSensitivityAnalysisRun({ + scope: "all", + status: "running", + total: 2, + suggestedSensitive: 0, + suggestedNonSensitive: 0, + unknown: 2, + finishedAt: null, + }); + const completedRun = makeSensitivityAnalysisRun({ scope: "all", total: 2 }); + let activityEvents = [{ + runId: runningRun.id, + sequence: 1, + level: "info", + message: "Scanning source columns", + createdAt: "2026-08-28T11:00:00Z", + }]; + server.use( + http.get("/api/catalog/sensitive-data-suggestion-runs", () => ( + HttpResponse.json(analysisStarted ? [runningRun] : []) + )), + http.get("/api/catalog/sensitive-data-suggestion-runs/:runId", () => ( + HttpResponse.json(runningRun) + )), + http.get("/api/catalog/sensitive-data-suggestion-runs/:runId/events-list", () => ( + HttpResponse.json(activityEvents) + )), + http.post("/api/catalog/databases/:databaseId/sensitive-data-suggestions", async () => { + analysisStarted = true; + await analysisGate; + return HttpResponse.json({ run: completedRun, suggestions: [] }); + }), + ); + renderPage({ rows: [makeDatabase()] }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); + const launch = user.click(screen.getByRole("button", { name: "Analyze sensitive fields" })); + + await waitFor(() => expect(analysisStarted).toBe(true)); + const progress = await screen.findByRole("dialog", { name: "Sensitivity analysis history" }); + expect(within(progress).getByRole("heading", { name: "Starting sensitivity analysis" })).toBeVisible(); + expect(within(progress).getByRole("progressbar", { name: "Sensitivity analysis is starting" })) + .toBeVisible(); + expect(within(progress).getByText("Preparing sensitivity analysis…")).toBeVisible(); + expect(await within(progress).findByRole("heading", { name: "Running" }, { timeout: 2_000 })).toBeVisible(); + expect(await within(progress).findByText("Scanning source columns")).toBeVisible(); + + const eventLog = within(progress).getByRole("log", { name: "Sensitivity analysis events" }); + Object.defineProperties(eventLog, { + clientHeight: { configurable: true, value: 100 }, + scrollHeight: { configurable: true, get: () => activityEvents.length * 100 }, + scrollTop: { configurable: true, value: 0, writable: true }, + }); + activityEvents = [...activityEvents, { + runId: runningRun.id, + sequence: 2, + level: "info", + message: "Running local entity detection", + createdAt: "2026-08-28T11:00:01Z", + }]; + expect(await within(progress).findByText("Running local entity detection", {}, { timeout: 2_000 })).toBeVisible(); + await waitFor(() => expect(eventLog.scrollTop).toBe(eventLog.scrollHeight)); + + eventLog.scrollTop = 0; + fireEvent.scroll(eventLog); + const jumpToLatest = await within(progress).findByRole("button", { name: "Jump to latest" }); + activityEvents = [...activityEvents, { + runId: runningRun.id, + sequence: 3, + level: "info", + message: "Classifying the next table", + createdAt: "2026-08-28T11:00:02Z", + }]; + expect(await within(progress).findByText("Classifying the next table", {}, { timeout: 2_000 })).toBeVisible(); + expect(eventLog.scrollTop).toBe(0); + await user.click(jumpToLatest); + expect(eventLog.scrollTop).toBe(eventLog.scrollHeight); + expect(within(progress).queryByRole("button", { name: "Jump to latest" })).not.toBeInTheDocument(); + + releaseAnalysis(); + await launch; + expect(await screen.findByRole("dialog", { name: "Sensitive field review" })).toBeVisible(); +}); + test("requests database-level sensitivity analysis for the only selected database", async () => { const user = userEvent.setup(); let suggestionBody: unknown; @@ -2304,13 +2738,14 @@ test("allows a human downgrade and saves only explicit sensitivity changes", asy expect(patches).toHaveLength(0); await user.click(within(review).getByRole("checkbox", { name: "Protect patients.id" })); - await user.click(within(review).getByRole("button", { name: "Save 1" })); + await user.click(within(review).getByRole("button", { name: "Save all 1 change" })); await waitFor(() => expect(patches).toEqual([{ columnId: nameColumn.id, body: { version: nameColumn.version, sensitive: false, + sensitivityReason: null, }, }])); expect(await screen.findByText("Saved 1 sensitive flag")).toBeVisible(); diff --git a/frontend/src/shell/DatabaseManagementPage.tsx b/frontend/src/shell/DatabaseManagementPage.tsx index 45c1aa91..511bfda3 100644 --- a/frontend/src/shell/DatabaseManagementPage.tsx +++ b/frontend/src/shell/DatabaseManagementPage.tsx @@ -6,11 +6,20 @@ import { useState, } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { History, RefreshCw } from "lucide-react"; +import { CheckCircle2, CircleAlert, History, RefreshCw } from "lucide-react"; import { toast } from "sonner"; import { Button } from "../components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "../components/ui/dialog"; import { ApiError, apiErrorMessage } from "../api/client"; import { + consolidateCatalogDescriptions, createCatalogDatabase, deleteCatalogDatabase, deleteCatalogDatabaseMetadata, @@ -26,6 +35,7 @@ import { testCatalogDatabase, updateCatalogDatabase, type CatalogDatabase, + type CatalogDatabaseTestResult, type CatalogColumn, type CatalogDatabaseMetadataDeleteTarget, type CatalogSecretName, @@ -107,6 +117,35 @@ interface FormSource { version: number; } +interface ConnectionTestResult { + outcome: "success" | "failure"; + title: string; + description: string; +} + +function mergeConnectionTestResult( + source: CatalogDatabase, + tested: CatalogDatabaseTestResult, +): CatalogDatabase { + return { + ...source, + ...tested, + workspaceName: source.workspaceName, + workspaceDescription: source.workspaceDescription, + workspaceAvailable: source.workspaceAvailable, + workspaceRevision: source.workspaceRevision, + workspaceEvidence: source.workspaceEvidence, + runtimeBinding: source.runtimeBinding, + activeSyncRun: source.activeSyncRun, + }; +} + +function databaseDisplayName(database: CatalogDatabase): string { + return database.workspaceName?.trim() + || database.databaseName?.trim() + || database.workspaceId; +} + function isStaleError(error: unknown): boolean { return error instanceof ApiError && error.code === "database_stale"; } @@ -181,6 +220,7 @@ export function DatabaseManagementPage({ const [stale, setStale] = useState(false); const [staleBannerOpen, setStaleBannerOpen] = useState(true); const [partialSecretFailure, setPartialSecretFailure] = useState(null); + const [connectionTestResult, setConnectionTestResult] = useState(null); const [tablesNavigationState, setTablesNavigationState] = useState({ dirty: false, busy: false, @@ -198,6 +238,7 @@ export function DatabaseManagementPage({ const [descriptionGenerationDrawerOpen, setDescriptionGenerationDrawerOpen] = useState(false); const [activeSensitivityAnalysisRun, setActiveSensitivityAnalysisRun] = useState(null); const [sensitivityAnalysisHistoryDrawerOpen, setSensitivityAnalysisHistoryDrawerOpen] = useState(false); + const [sensitivityAnalysisStarting, setSensitivityAnalysisStarting] = useState(false); const [sensitiveReview, setSensitiveReview] = useState<{ databaseId: string; scopeLabel: string; @@ -546,9 +587,11 @@ export function DatabaseManagementPage({ || busy ) return; + setConnectionTestResult(null); setBusyAction("test"); try { - const tested = await testCatalogDatabase(formSource.id, formSource.version); + const testResult = await testCatalogDatabase(formSource.id, formSource.version); + const tested = mergeConnectionTestResult(activeRow, testResult); setFormSource({ id: tested.id, version: tested.version }); cacheSavedRow(tested); const nextDraft = draftFrom(tested); @@ -556,13 +599,25 @@ export function DatabaseManagementPage({ setBaseline(configurationFingerprint(nextDraft)); await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); if (tested.connectionStatus === "reachable") { - toast.success("Database connection is reachable"); + setConnectionTestResult({ + outcome: "success", + title: "Connection test successful", + description: `${databaseDisplayName(tested)} is reachable.`, + }); } else { - toast.error(tested.lastErrorMessage ?? "Database connection failed"); + setConnectionTestResult({ + outcome: "failure", + title: "Connection test failed", + description: `${databaseDisplayName(tested)}: ${tested.lastErrorMessage ?? "The database could not be reached."}`, + }); } } catch (error) { if (isStaleError(error)) markStale(); - else toast.error(apiErrorMessage(error)); + setConnectionTestResult({ + outcome: "failure", + title: "Connection test failed", + description: apiErrorMessage(error), + }); } finally { setBusyAction(null); } @@ -740,13 +795,41 @@ export function DatabaseManagementPage({ }, [activeRow?.id, activeSyncRun]); const testSelected = useCallback(async (selected: CatalogDatabase[]) => { + setConnectionTestResult(null); try { - const tested = await Promise.all(selected.map((row) => testCatalogDatabase(row.id!, row.version))); + const tested = await Promise.all(selected.map(async (row) => mergeConnectionTestResult( + row, + await testCatalogDatabase(row.id!, row.version), + ))); for (const row of tested) cacheSavedRow(row); await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); - toast.success(`${tested.length} connection${tested.length === 1 ? "" : "s"} tested`); + const failed = tested.filter((row) => row.connectionStatus !== "reachable"); + if (failed.length === 0) { + setConnectionTestResult({ + outcome: "success", + title: tested.length === 1 ? "Connection test successful" : "All connection tests successful", + description: tested.length === 1 + ? `${databaseDisplayName(tested[0])} is reachable.` + : `${tested.length} databases are reachable.`, + }); + } else { + const succeeded = tested.length - failed.length; + setConnectionTestResult({ + outcome: "failure", + title: tested.length === 1 + ? "Connection test failed" + : `${failed.length} of ${tested.length} connection tests failed`, + description: tested.length === 1 + ? `${databaseDisplayName(failed[0])}: ${failed[0].lastErrorMessage ?? "The database could not be reached."}` + : `${succeeded} succeeded. Failed: ${failed.map(databaseDisplayName).join(", ")}.`, + }); + } } catch (error) { - toast.error(apiErrorMessage(error)); + setConnectionTestResult({ + outcome: "failure", + title: "Connection test failed", + description: apiErrorMessage(error), + }); throw error; } }, [cacheSavedRow, queryClient]); @@ -786,6 +869,27 @@ export function DatabaseManagementPage({ } }, [rememberDescriptionGenerationRun, selectedMetadataModel]); + const consolidateDatabaseColumnDescriptions = useCallback(async ( + selected: CatalogDatabase[], + ) => { + const database = selected.length === 1 ? selected[0] : undefined; + if (!database?.id) return; + try { + const result = await consolidateCatalogDescriptions(database.id, "database_columns"); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["catalog-tables", database.id] }), + queryClient.invalidateQueries({ queryKey: ["catalog-columns", database.id] }), + invalidateCatalogMetrics(), + ]); + toast.success( + `Copied ${result.copied} column description${result.copied === 1 ? "" : "s"}; skipped ${result.skipped}`, + ); + } catch (error) { + toast.error(apiErrorMessage(error)); + throw error; + } + }, [invalidateCatalogMetrics, queryClient]); + const requestSensitiveSuggestions = useCallback(async ( database: CatalogDatabase, selection: SensitivityAnalysisRequest, @@ -795,6 +899,11 @@ export function DatabaseManagementPage({ toast.error("The selected database is not configured, so sensitivity analysis was not started."); throw new Error("database is not configured"); } + setSensitiveReview(null); + setActiveSensitivityAnalysisRun(null); + setDescriptionGenerationDrawerOpen(false); + setSensitivityAnalysisStarting(true); + setSensitivityAnalysisHistoryDrawerOpen(true); try { const result = await runSensitivityAnalysis(database.id, selection); if (result.run) { @@ -804,6 +913,7 @@ export function DatabaseManagementPage({ (current = []) => [result.run, ...current.filter((item) => item.id !== result.run.id)], ); } + setSensitivityAnalysisHistoryDrawerOpen(false); setSensitiveReview({ databaseId: database.id, scopeLabel, suggestions: result.suggestions }); toast.success(`Prepared ${result.suggestions.length} local sensitivity assessment${result.suggestions.length === 1 ? "" : "s"} for review`); } catch (error) { @@ -811,6 +921,7 @@ export function DatabaseManagementPage({ throw error; } finally { await queryClient.invalidateQueries({ queryKey: SENSITIVE_DATA_SUGGESTION_HISTORY_QUERY_KEY }); + setSensitivityAnalysisStarting(false); } }, [queryClient]); @@ -961,6 +1072,41 @@ export function DatabaseManagementPage({ { id: screen.kind, label: screen.kind === "tables" ? "Tables" : "Relationships", current: true }, ] : [{ id: "databases", label: "Databases", current: true }]; + const connectionTestResultDialog = ( + { + if (!open) setConnectionTestResult(null); + }} + > + + +
+ +
+ {connectionTestResult?.title ?? "Connection test"} + + {connectionTestResult?.description} + +
+
+ + + + + + + ); const catalogDrawers = ( <> setSensitivityAnalysisHistoryDrawerOpen(false)} onRunUpdate={setActiveSensitivityAnalysisRun} /> @@ -999,6 +1146,7 @@ export function DatabaseManagementPage({ onClose={() => setSensitiveReview(null)} onSaved={sensitiveColumnsSaved} /> + {connectionTestResultDialog} ); @@ -1120,6 +1268,7 @@ export function DatabaseManagementPage({ selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null} descriptionGenerationActive={descriptionGenerationActive} onGenerateDescriptions={generateDatabaseDescriptions} + onConsolidateColumnDescriptions={consolidateDatabaseColumnDescriptions} onSuggestSensitive={suggestDatabaseSensitiveFields} sensitivityAnalysisRuns={sensitivityAnalysisRuns} onDeleteMetadataSelected={deleteSelectedMetadata} @@ -1328,6 +1477,7 @@ export function DatabaseManagementPage({ descriptionGenerationActive={descriptionGenerationActive} sensitivityAnalysisRuns={sensitivityAnalysisRuns} onGenerateDescriptions={generateDatabaseDescriptions} + onConsolidateColumnDescriptions={consolidateDatabaseColumnDescriptions} onSuggestSensitive={suggestDatabaseSensitiveFields} onDeleteMetadataSelected={deleteSelectedMetadata} /> @@ -1428,6 +1578,7 @@ export function DatabaseManagementPage({ open={sensitivityAnalysisHistoryDrawerOpen} databaseId={activeRow?.id ?? null} run={activeSensitivityAnalysisRun} + awaitingRun={sensitivityAnalysisStarting} onClose={() => setSensitivityAnalysisHistoryDrawerOpen(false)} onRunUpdate={setActiveSensitivityAnalysisRun} /> @@ -1440,6 +1591,7 @@ export function DatabaseManagementPage({ onClose={() => setSensitiveReview(null)} onSaved={sensitiveColumnsSaved} /> + {connectionTestResultDialog} ); } diff --git a/frontend/src/shell/WorkspaceManager.test.tsx b/frontend/src/shell/WorkspaceManager.test.tsx index 13634925..c50652d6 100644 --- a/frontend/src/shell/WorkspaceManager.test.tsx +++ b/frontend/src/shell/WorkspaceManager.test.tsx @@ -241,8 +241,9 @@ test("workspace-specific commands remain isolated until a workspace is selected" expect(screen.getByText(/exact version currently activated by ThothII/i)).toBeVisible(); expect(screen.getByText(/does not modify the repository/i)).toBeVisible(); expect(screen.getByText(/confirms that workspace.yaml and the directories/i)).toBeVisible(); - expect(screen.getByText(/reach the data warehouse and Evidence sources/i)).toBeVisible(); - expect(screen.getByText(/credentials only for the duration of the test/i)).toBeVisible(); + expect(screen.getByText(/database configured in Database Management/i)).toBeVisible(); + expect(screen.getByText(/Evidence source and the installation semantic services/i)).toBeVisible(); + expect(screen.getByText(/result is informational/i)).toBeVisible(); const databaseField = screen.getByText("Database").parentElement; expect(databaseField).not.toBeNull(); expect(databaseField).toHaveTextContent("engine: postgres"); diff --git a/frontend/src/shell/WorkspaceManager.tsx b/frontend/src/shell/WorkspaceManager.tsx index 7ccad58b..5d681809 100644 --- a/frontend/src/shell/WorkspaceManager.tsx +++ b/frontend/src/shell/WorkspaceManager.tsx @@ -570,7 +570,7 @@ export function WorkspaceManager({

Test workspace connections

-

Verifies that ThothII can reach the data warehouse and Evidence sources configured for this workspace. It decrypts credentials only for the duration of the test and deletes temporary files when the check finishes.

+

Verifies the database configured in Database Management, plus this workspace's Evidence source and the installation semantic services. The result is informational.

{connectionNotice && (

{connectionNotice} diff --git a/frontend/src/shell/activityScroll.ts b/frontend/src/shell/activityScroll.ts index 423455ac..dcc9604f 100644 --- a/frontend/src/shell/activityScroll.ts +++ b/frontend/src/shell/activityScroll.ts @@ -1,6 +1,52 @@ +import { + useCallback, + useEffect, + useLayoutEffect, + useRef, + useState, + type UIEventHandler, +} from "react"; + export function isNearBottom( el: Pick, threshold = 48, ): boolean { return el.scrollHeight - el.clientHeight - el.scrollTop <= threshold; } + +export function useTailFollowing({ + active = true, + resetKey, + tail, + threshold = 48, +}: { + active?: boolean; + resetKey?: unknown; + tail: unknown; + threshold?: number; +}) { + const viewportRef = useRef(null); + const [followingTail, setFollowingTail] = useState(true); + const scrollToTail = useCallback(() => { + const viewport = viewportRef.current; + if (!viewport) return; + viewport.scrollTop = viewport.scrollHeight; + }, []); + const handleScroll = useCallback>((event) => { + setFollowingTail(isNearBottom(event.currentTarget, threshold)); + }, [threshold]); + const jumpToTail = useCallback(() => { + setFollowingTail(true); + scrollToTail(); + }, [scrollToTail]); + + useEffect(() => { + if (active) setFollowingTail(true); + }, [active, resetKey]); + + useLayoutEffect(() => { + if (active && followingTail) scrollToTail(); + }, [active, followingTail, resetKey, scrollToTail, tail]); + + return { followingTail, handleScroll, jumpToTail, viewportRef }; +} diff --git a/frontend/src/shell/database-management/CatalogSyncDrawer.test.tsx b/frontend/src/shell/database-management/CatalogSyncDrawer.test.tsx new file mode 100644 index 00000000..a765700c --- /dev/null +++ b/frontend/src/shell/database-management/CatalogSyncDrawer.test.tsx @@ -0,0 +1,128 @@ +import { act, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { http, HttpResponse } from "msw"; +import { useState } from "react"; +import type { CatalogSyncEvent, CatalogSyncRun } from "../../api/catalog-databases"; +import { FakeEventSource } from "../../test/fakeEventSource"; +import { server } from "../../test/msw"; +import { CatalogSyncDrawer } from "./CatalogSyncDrawer"; + +const runningRun: CatalogSyncRun = { + id: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + databaseId: "11111111-1111-4111-8111-111111111111", + scope: "all", + tableIds: [], + state: "running", + phase: "scanning_columns", + requestedDatabaseVersion: 3, + plannedDiff: null, + confirmationToken: null, + counts: { tables: 2, columns: 12 }, + errorCode: null, + errorMessage: null, + cancelRequested: false, + createdAt: "2026-08-27T10:00:00Z", + startedAt: "2026-08-27T10:00:01Z", + updatedAt: "2026-08-27T10:00:02Z", + finishedAt: null, + heartbeatAt: "2026-08-27T10:00:02Z", +}; + +function event(sequence: number, message: string): CatalogSyncEvent { + return { + id: sequence, + runId: runningRun.id, + sequence, + level: "info", + eventType: sequence === 1 ? "started" : "scanning_columns", + message, + data: {}, + createdAt: `2026-08-27T10:00:0${sequence}Z`, + }; +} + +function renderDrawer() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: Infinity } }, + }); + + function Harness() { + const [runId, setRunId] = useState(runningRun.id); + return ( + undefined} + onRunChange={(run) => setRunId(run.id)} + onRunUpdate={() => undefined} + onCatalogChanged={() => undefined} + /> + ); + } + + return render( + + + , + ); +} + +const nativeEventSource = globalThis.EventSource; + +beforeEach(() => { + FakeEventSource.instances = []; + (globalThis as unknown as { EventSource: typeof EventSource }).EventSource = ( + FakeEventSource as unknown as typeof EventSource + ); +}); + +afterEach(() => { + (globalThis as unknown as { EventSource: typeof EventSource }).EventSource = nativeEventSource; +}); + +test("follows the synchronization log tail until the operator scrolls upward", async () => { + const user = userEvent.setup(); + let logHeight = 100; + server.use( + http.get("/api/catalog/sync-runs/:runId", () => HttpResponse.json(runningRun)), + http.get("/api/catalog/sync-runs/:runId/events-list", () => ( + HttpResponse.json([event(1, "Synchronization started")]) + )), + http.get("/api/catalog/databases/:databaseId/sync-runs", () => ( + HttpResponse.json([runningRun]) + )), + ); + renderDrawer(); + + const log = await screen.findByRole("log", { name: "Synchronization events" }); + expect(await within(log).findByText("Synchronization started")).toBeVisible(); + await waitFor(() => expect(FakeEventSource.instances).toHaveLength(1)); + Object.defineProperties(log, { + clientHeight: { configurable: true, value: 100 }, + scrollHeight: { configurable: true, get: () => logHeight }, + scrollTop: { configurable: true, value: 0, writable: true }, + }); + + logHeight = 200; + act(() => { + FakeEventSource.instances[0].emitNamed("log", event(2, "Reading source columns"), "2"); + }); + expect(await within(log).findByText("Reading source columns")).toBeVisible(); + await waitFor(() => expect(log.scrollTop).toBe(log.scrollHeight)); + + log.scrollTop = 0; + fireEvent.scroll(log); + const jumpToLatest = await screen.findByRole("button", { name: "Jump to latest" }); + logHeight = 300; + act(() => { + FakeEventSource.instances[0].emitNamed("log", event(3, "Reading source relationships"), "3"); + }); + expect(await within(log).findByText("Reading source relationships")).toBeVisible(); + expect(log.scrollTop).toBe(0); + + await user.click(jumpToLatest); + expect(log.scrollTop).toBe(log.scrollHeight); + expect(screen.queryByRole("button", { name: "Jump to latest" })).not.toBeInTheDocument(); +}); diff --git a/frontend/src/shell/database-management/CatalogSyncDrawer.tsx b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx index 2a508aad..744506db 100644 --- a/frontend/src/shell/database-management/CatalogSyncDrawer.tsx +++ b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx @@ -16,6 +16,7 @@ import { type CatalogSyncPhase, type CatalogSyncRun, } from "../../api/catalog-databases"; +import { useTailFollowing } from "../activityScroll"; import { FleetLedgerDrawer } from "./FleetLedgerShell"; interface Props { @@ -127,6 +128,17 @@ export function CatalogSyncDrawer({ retry: false, refetchInterval: run && terminal(run) ? false : 1_500, }); + const latestEventSequence = events.at(-1)?.sequence ?? 0; + const { + followingTail: followingEventTail, + handleScroll: handleEventLogScroll, + jumpToTail: jumpToLatestEvent, + viewportRef: eventLogRef, + } = useTailFollowing({ + active: open && Boolean(run), + resetKey: runId, + tail: latestEventSequence, + }); const mergeEvents = (incoming: CatalogSyncEvent[]) => { if (incoming.length === 0) return; @@ -416,9 +428,24 @@ export function CatalogSyncDrawer({

Live log

- {events.length} +
+ {!followingEventTail && events.length > 0 ? ( + + ) : null} + {events.length} +
-
+
{events.length === 0 ?

Waiting for events…

: events.map((event) => (

{new Date(event.createdAt).toLocaleTimeString()}{event.message} diff --git a/frontend/src/shell/database-management/DatabaseColumns.tsx b/frontend/src/shell/database-management/DatabaseColumns.tsx index b6499271..e8870a39 100644 --- a/frontend/src/shell/database-management/DatabaseColumns.tsx +++ b/frontend/src/shell/database-management/DatabaseColumns.tsx @@ -13,6 +13,7 @@ import { startCatalogSync, startDescriptionGenerationRun, updateCatalogColumnMetadata, + updateCatalogColumnSensitive, type CatalogColumn, type CatalogSyncRun, type CatalogTable, @@ -89,6 +90,76 @@ function SensitiveCell({ data, context }: ICellRendererParams void; + onGeneratedDescriptionChange: (value: string) => void; +}) { + const fieldClass = [ + "thot-column-metadata-field flex min-h-0 flex-col gap-1.5", + "text-sm font-semibold", + ].join(" "); + const editableClass = [ + "min-h-28 flex-1 resize-none rounded-md border border-input bg-card px-3 py-2", + "text-sm font-normal outline-none focus:border-primary/60 focus:ring-3 focus:ring-ring/15", + ].join(" "); + const readOnlyClass = [ + "min-h-28 flex-1 resize-none rounded-md border border-input bg-muted/35 px-3 py-2", + "text-sm font-normal", + ].join(" "); + return ( +

+