mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s
Details

Browse Source
This commit is contained in:
Codex
2026-09-06 17:49:35 +02:00
parent 8707ae1d46
commit cffa60772e
141 changed files with 5898 additions and 3015 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/app.ts
+27 -11
View File
@@ -76,6 +76,10 @@ import { CatalogLogicalRelationshipService } from "./catalog/logical-relationshi
import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
import { loadRuntimeModelCatalog, type RuntimeModelCatalog } from "./models/runtime-model-catalog.js";
import { createProductionWorkspacePreprocessingService } from "./workspace-maintenance.js";
import type { WorkspacePreprocessingService } from "./workspaces/preprocessing-service.js";
import { PreprocessingStateStore } from "./workspaces/preprocessing-state.js";
import { workspacePreprocessingRoutes } from "./routes/workspace-preprocessing.js";
export interface BuildAppDeps {
thtRunner?: ThtRunner;
@@ -89,6 +93,7 @@ export interface BuildAppDeps {
workspaceDiagnoser?: WorkspaceDiagnoser;
workspaceDatabaseTester?: WorkspaceDatabaseTester;
workspaceSecretStore?: WorkspaceSecretStore;
workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear">;
catalogRepository?: CatalogRepository;
catalogService?: CatalogService;
catalogPostgresAccess?: CatalogPostgresAccess;
@@ -156,6 +161,8 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
app.register(cookie);
app.register(rateLimit, { global: false });
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
const tht = deps?.thtRunner ?? new ThtRunner({
thtBin: config.thtBin,
harnessDir: config.harnessDir,
@@ -166,6 +173,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
secretsFile: config.secretsFile,
secretFiles: config.secretFiles,
workspaceSecretStore,
catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
@@ -174,9 +182,15 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
},
});
const workspacePreprocessingService = deps?.workspacePreprocessingService
?? createProductionWorkspacePreprocessingService({
config,
catalogRepository,
registry: workspaceRegistry,
workspaceSecretStore,
runner: tht as ThtRunner,
});
const hub = deps?.hub ?? new SseHub();
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
const runtimeModelCatalog = deps?.runtimeModelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
const mgr = deps?.mgr ?? new PiProcessManager(config, {
@@ -242,14 +256,6 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
?? new CatalogLogicalRelationshipService(catalogRepository);
const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider
?? (config.catalogDatabase === undefined
? undefined
: new EffectiveRelationshipSnapshotProvider(
catalogRepository,
catalogLogicalRelationshipService,
catalogOperationCoordinator,
));
const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
catalogPostgresAccess,
workspaceSecretStore,
@@ -457,7 +463,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
workspaceRuntimeSupport,
modelCatalog: runtimeModelCatalog,
maintenanceBarrier,
effectiveRelationships,
catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
});
app.post("/internal/maintenance/activate", async (req, reply) => {
try {
@@ -496,6 +502,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
secretStore: workspaceSecretStore,
testDatabaseConnection: workspaceDatabaseTester,
});
workspacePreprocessingRoutes(app, {
repository: catalogRepository,
registry: workspaceRegistry,
service: workspacePreprocessingService,
inputFingerprint: tht as ThtRunner,
readLatestJob: (workspaceId) => new PreprocessingStateStore({
dataRoot: config.dataRoot ?? "/data",
workspaceId,
}).readLatestJob(),
});
catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
catalogTableRoutes(app, {
repository: catalogRepository,
backend/src/catalog/description-generation-worker.ts
+41 -16
View File
@@ -59,10 +59,16 @@ const completionResponseSchema = z.object({
class InvalidModelJsonError extends Error {}
class InvalidModelSchemaError extends Error {}
class MissingModelTargetsError extends Error {}
interface DescriptionGenerationFailureTarget {
id: string;
reference: string;
}
class DescriptionGenerationBatchError extends Error {
constructor(
readonly failure: unknown,
readonly failedTargets: readonly { id: string; label: "Catalog Column" | "Catalog Table" }[],
readonly failedTargets: readonly DescriptionGenerationFailureTarget[],
) {
super("description generation batch failed");
}
@@ -138,7 +144,7 @@ type ParsedOutcome = z.infer<typeof outcomeSchema>;
interface DescriptionGenerationPlan {
columnTargets: SelectedColumnTarget[];
tableIds: string[];
tableTargets: Array<{ id: string; name: string }>;
}
interface DescriptionGenerationCounters {
@@ -164,10 +170,17 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
};
}
function failureTarget(target: SelectedTarget) {
function targetReference(target: SelectedTarget): string {
return target.kind === "column"
? { id: target.column.id, label: "Catalog Column" as const }
: { id: target.table.id, label: "Catalog Table" as const };
? `Column ${JSON.stringify(`${target.table.name}.${target.column.name}`)}`
: `Table ${JSON.stringify(target.table.name)}`;
}
function failureTarget(target: SelectedTarget): DescriptionGenerationFailureTarget {
return {
id: target.kind === "column" ? target.column.id : target.table.id,
reference: targetReference(target),
};
}
const NON_GENERATABLE_DESCRIPTION: Record<DescriptionGenerationRun["language"], string> = {
@@ -681,7 +694,7 @@ function safeFailure(error: unknown): string {
function batchFailureEvent(error: DescriptionGenerationBatchError): string {
const summary = safeFailure(error);
return error.failedTargets.length > 0
? `${summary} Affected ${error.failedTargets[0]!.label} target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.id).join(", ")}.`
? `${summary} Affected target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.reference).join(", ")}.`
: summary;
}
@@ -794,7 +807,7 @@ export class DescriptionGenerationWorker {
scope === "selected_columns" ? "column" : "table",
);
}
const total = plan.columnTargets.length + plan.tableIds.length;
const total = plan.columnTargets.length + plan.tableTargets.length;
if (total === 0 && (scope === "all" || scope === "missing")) {
throw new DescriptionGenerationNoEligibleTargetsError(scope);
}
@@ -934,12 +947,18 @@ export class DescriptionGenerationWorker {
};
await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
throwIfCancelled(signal);
if (plan.tableIds.length > 0) {
const tableTargets = await this.resolveTableTargets(run.databaseId, plan.tableIds);
if (plan.tableTargets.length > 0) {
const tableTargets = await this.resolveTableTargets(
run.databaseId,
plan.tableTargets.map((target) => target.id),
);
if (!tableTargets) {
throw new DescriptionGenerationBatchError(
new Error("selected tables changed during generation"),
plan.tableIds.map((id) => ({ id, label: "Catalog Table" })),
plan.tableTargets.map((target) => ({
id: target.id,
reference: `Table ${JSON.stringify(target.name)}`,
})),
);
}
await this.processTargets(run, database, tableTargets, model, counters, signal);
@@ -1093,8 +1112,8 @@ export class DescriptionGenerationWorker {
run.id,
"info",
outcome.outcome === "generated"
? `Generated description for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`
: `Stored non-generatable result for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`,
? `Generated description for ${targetReference(target)}.`
: `Stored non-generatable result for ${targetReference(target)}.`,
);
}
}
@@ -1188,16 +1207,22 @@ export class DescriptionGenerationWorker {
}
return {
columnTargets,
tableIds: tables
tableTargets: tables
.filter((table) => scope === "all" || !table.generatedDescription?.trim())
.map((table) => table.id),
.map((table) => ({ id: table.id, name: table.name })),
};
}
if (scope === "selected_tables") {
const tableById = new Map(tables.map((table) => [table.id, table]));
const selected = targetIds.map((tableId) => tableById.get(tableId));
if (selected.some((table) => table === undefined)) return undefined;
return { columnTargets: [], tableIds: [...targetIds] };
return {
columnTargets: [],
tableTargets: (selected as CatalogTable[]).map((table) => ({
id: table.id,
name: table.name,
})),
};
}
const byId = new Map<string, SelectedColumnTarget>();
@@ -1209,7 +1234,7 @@ export class DescriptionGenerationWorker {
const targets = targetIds.map((columnId) => byId.get(columnId));
return targets.some((target) => target === undefined)
? undefined
: { columnTargets: targets as SelectedColumnTarget[], tableIds: [] };
: { columnTargets: targets as SelectedColumnTarget[], tableTargets: [] };
}
private async resolveTableTargets(
backend/src/catalog/memory-repository.ts
+93 -5
View File
@@ -18,6 +18,8 @@ import {
type CatalogLogicalRelationshipCandidate,
type CatalogLogicalRelationshipContext,
type CatalogPhysicalRelationship,
type CatalogPreprocessingStartResult,
type CatalogPreprocessingClearResult,
type CatalogSchemaDiff,
type CatalogSyncCounts,
type CatalogSyncEvent,
@@ -72,6 +74,77 @@ export class MemoryCatalogRepository implements CatalogRepository {
const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
return value ? clone(value) : undefined;
}
async beginPreprocessing(
workspaceId: string,
inputFingerprint: string,
): Promise<CatalogPreprocessingStartResult> {
const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
if (!database) return { kind: "not_found" };
if (database.preprocessingStatus === "running") return { kind: "already_running" };
if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
const catalogBusy = [...this.syncRuns.values()].some((run) =>
run.databaseId === database.id
&& ["queued", "running", "awaiting_confirmation", "applying"].includes(run.state))
|| [...this.descriptionGenerationRuns.values()].some((run) =>
run.databaseId === database.id && ["queued", "running"].includes(run.status))
|| [...this.sensitivityAnalysisRuns.values()].some((run) =>
run.databaseId === database.id && ["queued", "running"].includes(run.status));
if (catalogBusy) return { kind: "catalog_busy" };
const now = new Date().toISOString();
const updated: WorkspaceDatabase = {
...database,
preprocessingStatus: "running",
preprocessingInputFingerprint: inputFingerprint,
preprocessedMetadataRevision: undefined,
preprocessingStartedAt: now,
preprocessingFinishedAt: undefined,
preprocessingErrorCode: undefined,
updatedAt: now,
};
this.records.set(database.id, updated);
return { kind: "started", database: clone(updated) };
}
async finishPreprocessing(
workspaceId: string,
metadataContentRevision: number,
inputFingerprint: string,
outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
): Promise<WorkspaceDatabase | undefined> {
const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
if (!database
|| database.preprocessingStatus !== "running"
|| database.metadataContentRevision !== metadataContentRevision
|| database.preprocessingInputFingerprint !== inputFingerprint) return undefined;
const updated: WorkspaceDatabase = {
...database,
preprocessingStatus: outcome.status,
preprocessedMetadataRevision: outcome.status === "succeeded"
? metadataContentRevision
: undefined,
preprocessingFinishedAt: new Date().toISOString(),
preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : undefined,
};
this.records.set(database.id, updated);
return clone(updated);
}
async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
if (!database) return { kind: "not_found" };
if (database.preprocessingStatus === "running") return { kind: "already_running" };
const now = new Date().toISOString();
const updated: WorkspaceDatabase = {
...database,
preprocessingStatus: "failed",
preprocessingInputFingerprint: undefined,
preprocessedMetadataRevision: undefined,
preprocessingStartedAt: undefined,
preprocessingFinishedAt: now,
preprocessingErrorCode: "derived_data_cleared",
updatedAt: now,
};
this.records.set(database.id, updated);
return { kind: "cleared", database: clone(updated) };
}
async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
if (databaseId !== undefined && !this.records.has(databaseId)) return undefined;
@@ -114,6 +187,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
createdAt: now,
updatedAt: now,
connectionStatus: "untested",
metadataContentRevision: 0,
preprocessingStatus: "failed",
};
this.records.set(record.id, record);
return clone(record);
@@ -286,12 +361,24 @@ export class MemoryCatalogRepository implements CatalogRepository {
return undefined;
}
const now = new Date().toISOString();
if (target === "database_columns") {
const targets = [...this.columns.values()].filter((column) => (
if (target === "database" || target === "database_columns") {
const tableTargets = target === "database"
? [...this.tables.values()].filter((table) => table.databaseId === databaseId)
: [];
const columnTargets = [...this.columns.values()].filter((column) => (
this.tables.get(column.tableId)?.databaseId === databaseId
));
const copied = targets.filter((column) => Boolean(column.generatedDescription?.trim()));
for (const column of copied) {
const copiedTables = tableTargets.filter((table) => Boolean(table.generatedDescription?.trim()));
const copiedColumns = columnTargets.filter((column) => Boolean(column.generatedDescription?.trim()));
for (const table of copiedTables) {
this.tables.set(table.id, {
...table,
description: table.generatedDescription,
version: table.version + 1,
updatedAt: now,
});
}
for (const column of copiedColumns) {
this.columns.set(column.id, {
...column,
description: column.generatedDescription,
@@ -299,7 +386,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
updatedAt: now,
});
}
return { copied: copied.length, skipped: targets.length - copied.length };
const copied = copiedTables.length + copiedColumns.length;
return { copied, skipped: tableTargets.length + columnTargets.length - copied };
}
if (selectedTargetIds.length === 0) return undefined;
if (target === "tables") {
backend/src/catalog/metadata-snapshot.ts
+144
View File
@@ -0,0 +1,144 @@
import type {
CatalogColumn,
CatalogLogicalRelationship,
CatalogPhysicalRelationship,
CatalogRepository,
CatalogTable,
} from "./types.js";
export type CatalogDescriptionSource = "curated" | "generated" | "source_comment";
export interface CatalogMetadataSnapshotColumn {
id: string;
name: string;
ordinalPosition: number;
dataType: string;
isNullable: boolean;
defaultExpression: string | null;
primaryKeyPosition: number | null;
sensitive: boolean;
description: string | null;
descriptionSource: CatalogDescriptionSource | null;
}
export interface CatalogMetadataSnapshotTable {
id: string;
name: string;
description: string | null;
descriptionSource: CatalogDescriptionSource | null;
columns: CatalogMetadataSnapshotColumn[];
}
export interface CatalogMetadataSnapshotRelationship {
id: string;
origin: "physical" | "generated" | "manual";
sourceTable: string;
sourceColumns: string[];
targetTable: string;
targetColumns: string[];
}
export interface CatalogMetadataSnapshot {
schemaVersion: 1;
workspaceId: string;
databaseId: string;
databaseName: string;
schemaName: string;
metadataContentRevision: number;
tables: CatalogMetadataSnapshotTable[];
relationships: CatalogMetadataSnapshotRelationship[];
}
function effectiveDescription(value: {
description: string | null;
generatedDescription: string | null;
sourceComment: string | null;
}): { description: string | null; descriptionSource: CatalogDescriptionSource | null } {
if (value.description?.trim()) {
return { description: value.description.trim(), descriptionSource: "curated" };
}
if (value.generatedDescription?.trim()) {
return { description: value.generatedDescription.trim(), descriptionSource: "generated" };
}
if (value.sourceComment?.trim()) {
return { description: value.sourceComment.trim(), descriptionSource: "source_comment" };
}
return { description: null, descriptionSource: null };
}
function snapshotColumn(column: CatalogColumn): CatalogMetadataSnapshotColumn {
return {
id: column.id,
name: column.name,
ordinalPosition: column.ordinalPosition,
dataType: column.dataType,
isNullable: column.isNullable,
defaultExpression: column.defaultExpression,
primaryKeyPosition: column.primaryKeyPosition,
sensitive: column.sensitive,
...effectiveDescription(column),
};
}
function snapshotRelationship(
relationship: CatalogPhysicalRelationship | CatalogLogicalRelationship,
): CatalogMetadataSnapshotRelationship {
const columns = [...relationship.columns].sort((left, right) => left.position - right.position);
return {
id: relationship.id,
origin: relationship.origin,
sourceTable: relationship.sourceTableName,
sourceColumns: columns.map((column) => column.sourceColumnName),
targetTable: relationship.targetTableName,
targetColumns: columns.map((column) => column.targetColumnName),
};
}
export async function buildCatalogMetadataSnapshot(
repository: CatalogRepository,
workspaceId: string,
expectedMetadataContentRevision: number,
): Promise<CatalogMetadataSnapshot> {
const database = await repository.getByWorkspace(workspaceId);
if (!database || database.preprocessingStatus !== "running") {
throw new Error("catalog preprocessing lease is not active");
}
if (database.metadataContentRevision !== expectedMetadataContentRevision) {
throw new Error("catalog metadata revision changed");
}
const catalogTables = await repository.listTables(database.id);
const tables: CatalogMetadataSnapshotTable[] = [];
for (const table of [...catalogTables].sort((left, right) => left.name.localeCompare(right.name))) {
const columns = await repository.listColumns(database.id, table.id);
tables.push({
id: table.id,
name: table.name,
...effectiveDescription(table),
columns: columns
.sort((left, right) => left.ordinalPosition - right.ordinalPosition || left.name.localeCompare(right.name))
.map(snapshotColumn),
});
}
const physical = await repository.listRelationships(database.id);
const logical = (await repository.listLogicalRelationships(database.id))
.filter((relationship) => relationship.status === "active");
const relationships = [...physical, ...logical]
.map(snapshotRelationship)
.sort((left, right) =>
left.sourceTable.localeCompare(right.sourceTable)
|| left.targetTable.localeCompare(right.targetTable)
|| left.id.localeCompare(right.id));
return {
schemaVersion: 1,
workspaceId,
databaseId: database.id,
databaseName: database.databaseName,
schemaName: database.schema,
metadataContentRevision: expectedMetadataContentRevision,
tables,
relationships,
};
}
backend/src/catalog/migrate.ts
+2
View File
@@ -15,6 +15,7 @@ import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
import * as canonicalModelIdsMigration from "./migrations/010_canonical_model_ids.js";
import * as localSensitivityAnalysisMigration from "./migrations/011_local_sensitivity_analysis.js";
import * as sensitivityReasonMigration from "./migrations/012_sensitivity_reason.js";
import * as catalogPreprocessingStateMigration from "./migrations/013_catalog_preprocessing_state.js";
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
const host = process.env.THT_CATALOG_DB_HOST;
@@ -52,6 +53,7 @@ const provider: MigrationProvider = {
"010_canonical_model_ids": canonicalModelIdsMigration,
"011_local_sensitivity_analysis": localSensitivityAnalysisMigration,
"012_sensitivity_reason": sensitivityReasonMigration,
"013_catalog_preprocessing_state": catalogPreprocessingStateMigration,
};
},
};
backend/src/catalog/migrations/013_catalog_preprocessing_state.ts
+212
View File
@@ -0,0 +1,212 @@
import { type Kysely, sql } from "kysely";
import type { CatalogDatabase } from "../repository.js";
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
await db.schema.alterTable("workspace_databases")
.addColumn("metadata_content_revision", "bigint", (column) => column.notNull().defaultTo(0))
.addColumn("preprocessing_status", "text", (column) => column.notNull().defaultTo("failed"))
.addColumn("preprocessing_input_fingerprint", "text")
.addColumn("preprocessed_metadata_revision", "bigint")
.addColumn("preprocessing_started_at", "timestamptz")
.addColumn("preprocessing_finished_at", "timestamptz")
.addColumn("preprocessing_error_code", "text")
.execute();
await sql`
alter table workspace_databases
add constraint workspace_databases_preprocessing_status_check
check (preprocessing_status in ('running', 'succeeded', 'failed'))
`.execute(db);
await sql`
create function catalog_metadata_write_guard()
returns trigger
language plpgsql
as $$
declare
resolved_database_id uuid;
current_status text;
relation_id uuid;
table_id uuid;
begin
if tg_table_name = 'catalog_tables' then
resolved_database_id := coalesce(new.database_id, old.database_id);
elsif tg_table_name = 'catalog_columns' then
table_id := coalesce(new.table_id, old.table_id);
select database_id into resolved_database_id from catalog_tables where id = table_id;
elsif tg_table_name = 'catalog_relationships' then
resolved_database_id := coalesce(new.database_id, old.database_id);
elsif tg_table_name = 'catalog_relationship_columns' then
relation_id := coalesce(new.relationship_id, old.relationship_id);
select database_id into resolved_database_id from catalog_relationships where id = relation_id;
elsif tg_table_name = 'catalog_logical_relationships' then
resolved_database_id := coalesce(new.database_id, old.database_id);
elsif tg_table_name = 'database_bindings' then
if tg_op = 'UPDATE' and not (
new.transport is distinct from old.transport
or new.host is distinct from old.host
or new.port is distinct from old.port
or new.username is distinct from old.username
or new.base_url is distinct from old.base_url
or new.rest_path is distinct from old.rest_path
or new.rest_auth is distinct from old.rest_auth
or new.tls_servername is distinct from old.tls_servername
or new.ssh_host is distinct from old.ssh_host
or new.ssh_port is distinct from old.ssh_port
or new.ssh_username is distinct from old.ssh_username
or new.ssh_target_host is distinct from old.ssh_target_host
or new.ssh_target_port is distinct from old.ssh_target_port
) then
return new;
end if;
resolved_database_id := coalesce(new.database_id, old.database_id);
end if;
if resolved_database_id is null then
if tg_op = 'DELETE' then return old; else return new; end if;
end if;
select preprocessing_status into current_status
from workspace_databases
where id = resolved_database_id
for update;
if current_status = 'running' then
raise exception 'catalog preprocessing is running'
using errcode = '55000';
end if;
update workspace_databases
set metadata_content_revision = metadata_content_revision + 1,
preprocessing_status = 'failed',
preprocessing_finished_at = now(),
preprocessing_error_code = 'catalog_changed',
updated_at = now()
where id = resolved_database_id;
if tg_op = 'DELETE' then return old; else return new; end if;
end;
$$
`.execute(db);
for (const table of [
"catalog_tables",
"catalog_columns",
"catalog_relationships",
"catalog_relationship_columns",
"catalog_logical_relationships",
"database_bindings",
]) {
await sql.raw(`
create trigger ${table}_metadata_write_guard
before insert or update or delete on ${table}
for each row execute function catalog_metadata_write_guard()
`).execute(db);
}
await sql`
create function catalog_database_configuration_guard()
returns trigger
language plpgsql
as $$
begin
if new.workspace_id is distinct from old.workspace_id
or new.engine is distinct from old.engine
or new.database_name is distinct from old.database_name
or new.schema_name is distinct from old.schema_name then
if old.preprocessing_status = 'running' then
raise exception 'catalog preprocessing is running'
using errcode = '55000';
end if;
new.metadata_content_revision := old.metadata_content_revision + 1;
new.preprocessing_status := 'failed';
new.preprocessing_finished_at := now();
new.preprocessing_error_code := 'catalog_changed';
end if;
return new;
end;
$$
`.execute(db);
await sql`
create trigger workspace_databases_configuration_guard
before update of workspace_id, engine, database_name, schema_name on workspace_databases
for each row execute function catalog_database_configuration_guard()
`.execute(db);
await sql`
create function catalog_operation_start_guard()
returns trigger
language plpgsql
as $$
declare
current_status text;
starting boolean;
begin
if tg_table_name = 'catalog_sync_runs' then
starting := new.state in ('queued', 'running', 'awaiting_confirmation', 'applying');
else
starting := new.status in ('queued', 'running');
end if;
if not starting then
return new;
end if;
select preprocessing_status into current_status
from workspace_databases
where id = new.database_id
for update;
if current_status = 'running' then
raise exception 'catalog preprocessing is running'
using errcode = '55000';
end if;
return new;
end;
$$
`.execute(db);
for (const table of [
"catalog_sync_runs",
"description_generation_runs",
"sensitive_data_suggestion_runs",
]) {
await sql.raw(`
create trigger ${table}_operation_start_guard
before insert or update on ${table}
for each row execute function catalog_operation_start_guard()
`).execute(db);
}
}
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
for (const table of [
"catalog_sync_runs",
"description_generation_runs",
"sensitive_data_suggestion_runs",
]) {
await sql.raw(`drop trigger if exists ${table}_operation_start_guard on ${table}`).execute(db);
}
await sql`drop function if exists catalog_operation_start_guard()`.execute(db);
await sql`drop trigger if exists workspace_databases_configuration_guard on workspace_databases`.execute(db);
await sql`drop function if exists catalog_database_configuration_guard()`.execute(db);
for (const table of [
"catalog_tables",
"catalog_columns",
"catalog_relationships",
"catalog_relationship_columns",
"catalog_logical_relationships",
"database_bindings",
]) {
await sql.raw(`drop trigger if exists ${table}_metadata_write_guard on ${table}`).execute(db);
}
await sql`drop function if exists catalog_metadata_write_guard()`.execute(db);
await db.schema.alterTable("workspace_databases")
.dropConstraint("workspace_databases_preprocessing_status_check").execute();
for (const column of [
"preprocessing_error_code",
"preprocessing_finished_at",
"preprocessing_started_at",
"preprocessed_metadata_revision",
"preprocessing_input_fingerprint",
"preprocessing_status",
"metadata_content_revision",
]) {
await sql.raw(`alter table workspace_databases drop column ${column}`).execute(db);
}
}
backend/src/catalog/repository.ts
+146 -6
View File
@@ -27,6 +27,8 @@ import {
type CatalogLogicalRelationshipCandidate,
type CatalogLogicalRelationshipContext,
type CatalogPhysicalRelationship,
type CatalogPreprocessingStartResult,
type CatalogPreprocessingClearResult,
type CatalogSchemaDiff,
type CatalogSyncCounts,
type CatalogSyncEvent,
@@ -54,6 +56,11 @@ import {
} from "./types.js";
type Timestamp = ColumnType<Date, Date | string | undefined, Date | string>;
type NullableTimestamp = ColumnType<
Date | null,
Date | string | null | undefined,
Date | string | null
>;
interface WorkspaceDatabaseTable {
id: string;
@@ -66,6 +73,13 @@ interface WorkspaceDatabaseTable {
updatedAt: Timestamp;
schemaSyncedVersion: number | null;
schemaSyncedAt: Timestamp | null;
metadataContentRevision: Generated<number>;
preprocessingStatus: Generated<WorkspaceDatabase["preprocessingStatus"]>;
preprocessingInputFingerprint: Generated<string | null>;
preprocessedMetadataRevision: Generated<number | null>;
preprocessingStartedAt: NullableTimestamp;
preprocessingFinishedAt: NullableTimestamp;
preprocessingErrorCode: Generated<string | null>;
}
interface DatabaseBindingTable {
@@ -325,6 +339,19 @@ function serialize(row: JoinedRow): WorkspaceDatabase {
lastErrorMessage: present(row.lastErrorMessage),
schemaSyncedVersion: present(row.schemaSyncedVersion),
schemaSyncedAt: row.schemaSyncedAt == null ? undefined : new Date(row.schemaSyncedAt).toISOString(),
metadataContentRevision: Number(row.metadataContentRevision),
preprocessingStatus: row.preprocessingStatus,
preprocessingInputFingerprint: present(row.preprocessingInputFingerprint),
preprocessedMetadataRevision: row.preprocessedMetadataRevision == null
? undefined
: Number(row.preprocessedMetadataRevision),
preprocessingStartedAt: row.preprocessingStartedAt == null
? undefined
: new Date(row.preprocessingStartedAt).toISOString(),
preprocessingFinishedAt: row.preprocessingFinishedAt == null
? undefined
: new Date(row.preprocessingFinishedAt).toISOString(),
preprocessingErrorCode: present(row.preprocessingErrorCode),
};
}
@@ -477,6 +504,98 @@ export class KyselyCatalogRepository implements CatalogRepository {
return id ? await this.get(id.id) : undefined;
}
async beginPreprocessing(
workspaceId: string,
inputFingerprint: string,
): Promise<CatalogPreprocessingStartResult> {
return await this.db.transaction().execute(async (trx) => {
const database = await trx.selectFrom("workspaceDatabases")
.selectAll()
.where("workspaceId", "=", workspaceId)
.forUpdate()
.executeTakeFirst();
if (!database) return { kind: "not_found" };
if (database.preprocessingStatus === "running") return { kind: "already_running" };
if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
const activeSync = await trx.selectFrom("catalogSyncRuns")
.select("id")
.where("databaseId", "=", database.id)
.where("state", "in", ["queued", "running", "awaiting_confirmation", "applying"])
.executeTakeFirst();
const activeDescriptions = await trx.selectFrom("descriptionGenerationRuns")
.select("id")
.where("databaseId", "=", database.id)
.where("status", "in", ["queued", "running"])
.executeTakeFirst();
const activeSensitivity = await trx.selectFrom("sensitiveDataSuggestionRuns")
.select("id")
.where("databaseId", "=", database.id)
.where("status", "=", "running")
.executeTakeFirst();
if (activeSync || activeDescriptions || activeSensitivity) return { kind: "catalog_busy" };
await trx.updateTable("workspaceDatabases")
.set({
preprocessingStatus: "running",
preprocessingInputFingerprint: inputFingerprint,
preprocessedMetadataRevision: null,
preprocessingStartedAt: sql`now()`,
preprocessingFinishedAt: null,
preprocessingErrorCode: null,
updatedAt: sql`now()`,
})
.where("id", "=", database.id)
.execute();
return { kind: "started", database: (await selectOne(trx, database.id))! };
});
}
async finishPreprocessing(
workspaceId: string,
metadataContentRevision: number,
inputFingerprint: string,
outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
): Promise<WorkspaceDatabase | undefined> {
const result = await this.db.updateTable("workspaceDatabases")
.set({
preprocessingStatus: outcome.status,
preprocessedMetadataRevision: outcome.status === "succeeded" ? metadataContentRevision : null,
preprocessingFinishedAt: sql`now()`,
preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : null,
updatedAt: sql`now()`,
})
.where("workspaceId", "=", workspaceId)
.where("preprocessingStatus", "=", "running")
.where("metadataContentRevision", "=", metadataContentRevision)
.where("preprocessingInputFingerprint", "=", inputFingerprint)
.returning("id")
.executeTakeFirst();
return result ? await this.get(result.id) : undefined;
}
async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
return await this.db.transaction().execute(async (trx) => {
const database = await trx.selectFrom("workspaceDatabases")
.select(["id", "preprocessingStatus"])
.where("workspaceId", "=", workspaceId)
.forUpdate()
.executeTakeFirst();
if (!database) return { kind: "not_found" };
if (database.preprocessingStatus === "running") return { kind: "already_running" };
await trx.updateTable("workspaceDatabases").set({
preprocessingStatus: "failed",
preprocessingInputFingerprint: null,
preprocessedMetadataRevision: null,
preprocessingStartedAt: null,
preprocessingFinishedAt: sql`now()`,
preprocessingErrorCode: "derived_data_cleared",
updatedAt: sql`now()`,
}).where("id", "=", database.id).execute();
return { kind: "cleared", database: (await selectOne(trx, database.id))! };
});
}
async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
const result = await sql<CatalogMetricsRow>`
WITH requested_database AS (
@@ -763,7 +882,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
targetIds: readonly string[],
): Promise<CatalogDescriptionConsolidationCounts | undefined> {
const selectedTargetIds = [...new Set(targetIds)];
if (target !== "database_columns" && selectedTargetIds.length === 0) return undefined;
if (target !== "database" && target !== "database_columns" && selectedTargetIds.length === 0) {
return undefined;
}
return await this.db.transaction().execute(async (trx) => {
const database = await trx.selectFrom("workspaceDatabases").select("id")
.where("id", "=", databaseId).forUpdate().executeTakeFirst();
@@ -793,8 +914,23 @@ export class KyselyCatalogRepository implements CatalogRepository {
};
}
const tableRows = await trx.selectFrom("catalogTables").select("id")
.where("databaseId", "=", databaseId).execute();
const tableRows = await trx.selectFrom("catalogTables").select(["id", "generatedDescription"])
.where("databaseId", "=", databaseId)
.orderBy("id")
.forUpdate()
.execute();
const copiedTableIds = target === "database"
? tableRows
.filter((row) => Boolean(row.generatedDescription?.trim()))
.map((row) => row.id)
: [];
if (copiedTableIds.length > 0) {
await trx.updateTable("catalogTables").set({
description: sql`generated_description`,
version: sql`version + 1`,
updatedAt: sql`now()`,
}).where("id", "in", copiedTableIds).execute();
}
const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
.select(["id", "generatedDescription"])
.where("tableId", "in", tableRows.map((table) => table.id))
@@ -812,7 +948,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
version: sql`version + 1`,
updatedAt: sql`now()`,
});
update = target === "database_columns"
update = target === "database" || target === "database_columns"
? update
.where("tableId", "in", tableRows.map((table) => table.id))
.where(sql<boolean>`nullif(btrim(generated_description), '') is not null`)
@@ -820,8 +956,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
await update.execute();
}
return {
copied: copiedIds.length,
skipped: rows.length - copiedIds.length,
copied: copiedTableIds.length + copiedIds.length,
skipped: (target === "database" ? tableRows.length : 0) - copiedTableIds.length
+ rows.length - copiedIds.length,
};
});
}
@@ -1835,6 +1972,9 @@ export class UnavailableCatalogRepository implements CatalogRepository {
async list(): Promise<WorkspaceDatabase[]> { return this.fail(); }
async get(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
async getByWorkspace(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
async beginPreprocessing(): Promise<CatalogPreprocessingStartResult> { return this.fail(); }
async finishPreprocessing(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
async clearPreprocessing(): Promise<CatalogPreprocessingClearResult> { return this.fail(); }
async getCatalogMetrics(): Promise<CatalogMetrics | undefined> { return this.fail(); }
async create(): Promise<WorkspaceDatabase> { return this.fail(); }
async update(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
backend/src/catalog/runtime-binding.ts
+140
View File
@@ -0,0 +1,140 @@
import { dirname } from "node:path";
import { resolveRuntimeBindings, type RuntimeBindings } from "../workspaces/bindings.js";
import { buildInstallationContract, type InstallationSuffix } from "../workspaces/contracts.js";
import {
discoverWorkspaceSecretRequirements,
} from "../workspaces/secret-requirements.js";
import type {
WorkspaceSecretMaterialization,
WorkspaceSecretStore,
} from "../workspaces/secret-store.js";
import {
validateWorkspaceDescriptor,
type WorkspaceDescriptor,
} from "../workspaces/schema.js";
import { CATALOG_SECRET_IDS } from "./secrets.js";
import type { WorkspaceDatabase } from "./types.js";
export interface CatalogRuntimeBindingLease {
workspace: WorkspaceDescriptor;
bindings: RuntimeBindings;
release(): void;
}
const CATALOG_SECRET_BY_SUFFIX: Readonly<Partial<Record<InstallationSuffix, string>>> = {
PASSWORD_FILE: CATALOG_SECRET_IDS.password,
API_KEY_FILE: CATALOG_SECRET_IDS.apiKey,
TLS_CA_FILE: CATALOG_SECRET_IDS.tlsCa,
SSH_PRIVATE_KEY_FILE: CATALOG_SECRET_IDS.sshPrivateKey,
SSH_KNOWN_HOSTS_FILE: CATALOG_SECRET_IDS.sshKnownHosts,
};
function runtimeWorkspace(
workspace: WorkspaceDescriptor,
database: WorkspaceDatabase,
): WorkspaceDescriptor {
if (workspace.workspace.id !== database.workspaceId) {
throw new Error("Catalog database binding does not belong to the workspace");
}
const { dwh: _legacyDwh, diagnostics: _legacyDiagnostics, ...descriptor } = workspace;
const binding = database.binding;
return validateWorkspaceDescriptor({
...descriptor,
dwh: {
engine: "postgres",
database: database.databaseName,
schema: database.schema,
...(binding.port === undefined ? {} : { port: binding.port }),
supported_transports: [binding.transport],
},
...(binding.transport === "rest_api" ? {
diagnostics: {
dwh_rest: {
method: "GET",
path: binding.restPath ?? "/health",
auth: binding.restAuth ?? "bearer",
response: { database: "database", schema: "schema" },
},
},
} : {}),
});
}
function setIfDefined(
environment: NodeJS.ProcessEnv,
name: string | undefined,
value: string | number | undefined,
): void {
if (name !== undefined && value !== undefined && value !== "") environment[name] = String(value);
}
/**
* Project one PostgreSQL Catalog row into the legacy-shaped configuration consumed by the
* Python runtime. The authored workspace remains database-free; this object exists only for
* the lifetime of a backend-owned runtime lease.
*/
export function resolveCatalogRuntimeBinding(options: {
workspace: WorkspaceDescriptor;
database: WorkspaceDatabase;
environment: NodeJS.ProcessEnv;
secretRoots: readonly string[];
secretStore: WorkspaceSecretStore;
}): CatalogRuntimeBindingLease {
const workspace = runtimeWorkspace(options.workspace, options.database);
const evidenceRequirements = discoverWorkspaceSecretRequirements(
options.workspace,
options.environment,
).filter(({ connector }) => connector === "evidence");
const catalogSecretIds = Object.values(CATALOG_SECRET_IDS);
let materialization: WorkspaceSecretMaterialization | undefined;
try {
materialization = options.secretStore.materialize(
options.database.workspaceId,
[...catalogSecretIds, ...evidenceRequirements.map(({ id }) => id)],
);
const environment: NodeJS.ProcessEnv = { ...options.environment };
const roots = new Set(options.secretRoots);
for (const path of materialization.files.values()) roots.add(dirname(path));
const variables = buildInstallationContract(workspace).variables;
const variable = (role: "DWH" | "EVIDENCE", suffix: InstallationSuffix) => (
variables.find((candidate) => candidate.role === role && candidate.suffix === suffix)?.name
);
const binding = options.database.binding;
setIfDefined(environment, variable("DWH", "TRANSPORT"), binding.transport);
setIfDefined(environment, variable("DWH", "HOST"), binding.host);
setIfDefined(environment, variable("DWH", "PORT"), binding.port);
setIfDefined(environment, variable("DWH", "BASE_URL"), binding.baseUrl);
setIfDefined(environment, variable("DWH", "USER"), binding.username);
setIfDefined(environment, variable("DWH", "SSH_HOST"), binding.sshHost);
setIfDefined(environment, variable("DWH", "SSH_PORT"), binding.sshPort);
setIfDefined(environment, variable("DWH", "SSH_USER"), binding.sshUsername);
setIfDefined(environment, variable("DWH", "SSH_TARGET_HOST"), binding.sshTargetHost);
setIfDefined(environment, variable("DWH", "SSH_TARGET_PORT"), binding.sshTargetPort);
for (const [suffix, secretId] of Object.entries(CATALOG_SECRET_BY_SUFFIX) as Array<[
InstallationSuffix,
string,
]>) {
setIfDefined(environment, variable("DWH", suffix), materialization.files.get(secretId));
}
for (const requirement of evidenceRequirements) {
setIfDefined(environment, requirement.variable, materialization.files.get(requirement.id));
}
const bindings = resolveRuntimeBindings(workspace, environment, [...roots]);
let released = false;
return {
workspace,
bindings,
release: () => {
if (released) return;
released = true;
materialization?.release();
},
};
} catch (error) {
materialization?.release();
throw error;
}
}
backend/src/catalog/service.ts
+26 -54
View File
@@ -1,5 +1,3 @@
import { buildInstallationContract } from "../workspaces/contracts.js";
import { resolveBinding } from "../workspaces/bindings.js";
import type { WorkspaceRegistry } from "../workspaces/registry.js";
import type { WorkspaceDescriptor } from "../workspaces/schema.js";
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
@@ -45,60 +43,33 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
secrets: Record<CatalogSecretName, boolean>;
}
function bindingValue(workspace: WorkspaceDescriptor, values: Record<string, string>, suffix: string) {
const variable = buildInstallationContract(workspace).variables.find((entry) => (
entry.role === "DWH" && entry.suffix === suffix
));
return variable ? values[variable.name] : undefined;
}
function numeric(value: string | undefined): number | undefined {
if (!value) return undefined;
const parsed = Number(value);
return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65_535 ? parsed : undefined;
}
function yamlBinding(workspace: WorkspaceDescriptor, secretRoots: readonly string[]): DatabaseBinding {
const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
const value = (suffix: string) => bindingValue(workspace, effective.values, suffix);
return {
transport: effective.transport,
host: value("HOST"),
port: numeric(value("PORT")) ?? workspace.dwh.port,
username: value("USER"),
baseUrl: value("BASE_URL"),
restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
restAuth: workspace.diagnostics?.dwh_rest?.auth ?? "bearer",
tlsServername: value("TLS_SERVERNAME"),
sshHost: value("SSH_HOST"),
sshPort: numeric(value("SSH_PORT")),
sshUsername: value("SSH_USER"),
sshTargetHost: value("SSH_TARGET_HOST"),
sshTargetPort: numeric(value("SSH_TARGET_PORT")),
};
}
function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<CatalogSecretName, boolean> {
return Object.fromEntries(Object.entries(CATALOG_SECRET_IDS).map(([name, id]) => (
[name, store.has(workspaceId, id)]
))) as Record<CatalogSecretName, boolean>;
}
function workspaceRuntimeState(
function databaseRuntimeState(
store: WorkspaceSecretStore,
workspace: WorkspaceDescriptor,
secretRoots: readonly string[],
database: WorkspaceDatabase,
): NonNullable<CatalogListItem["runtimeBinding"]> {
const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
const secretVariables = new Set(requirements.map(({ variable }) => variable));
const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
const configurationRequired = requirements.some(({ id, required }) => (
required && !store.has(workspace.workspace.id, id)
)) || effective.missing.some((variable) => !secretVariables.has(variable));
const { binding, workspaceId } = database;
const configured = (id: string) => store.has(workspaceId, id);
const connectionComplete = binding.transport === "postgres_direct"
? Boolean(binding.host && binding.port && binding.username && configured(CATALOG_SECRET_IDS.password))
: binding.transport === "rest_api"
? Boolean(binding.baseUrl && (binding.restAuth === "none" || configured(CATALOG_SECRET_IDS.apiKey)))
: Boolean(
binding.username && binding.sshHost && binding.sshPort && binding.sshUsername
&& binding.sshTargetHost && binding.sshTargetPort
&& configured(CATALOG_SECRET_IDS.password)
&& configured(CATALOG_SECRET_IDS.sshPrivateKey)
&& configured(CATALOG_SECRET_IDS.sshKnownHosts),
);
return {
transport: effective.transport,
configurationState: configurationRequired ? "configuration_required" : "ready",
sessionTransportSupported: effective.transport !== "ssh_tunnel",
transport: binding.transport,
configurationState: connectionComplete ? "ready" : "configuration_required",
sessionTransportSupported: binding.transport !== "ssh_tunnel",
};
}
@@ -145,17 +116,18 @@ export class CatalogService {
const active = await Promise.all(workspaces.map(async (entry) => {
const database = byWorkspace.get(entry.id);
const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
const base = database ?? {
workspaceId: entry.id,
engine: "postgres" as const,
databaseName: workspace.dwh.database,
schema: workspace.dwh.schema,
databaseName: "",
schema: "",
version: 0,
createdAt: "",
updatedAt: "",
binding: runtimeDatabaseBinding,
binding: { transport: "postgres_direct" as const },
connectionStatus: "untested" as const,
metadataContentRevision: 0,
preprocessingStatus: "failed" as const,
};
return {
...base,
@@ -167,7 +139,7 @@ export class CatalogService {
blob: entry.revision.blob,
},
workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
runtimeBinding: database ? databaseRuntimeState(this.secretStore, database) : null,
configured: database !== undefined,
secrets: secretState(this.secretStore, entry.id),
};
@@ -195,13 +167,13 @@ export class CatalogService {
}
async normalizeInput(input: DatabaseConfigurationInput): Promise<DatabaseConfigurationInput> {
const workspace = await this.ensureWorkspace(input.workspaceId);
await this.ensureWorkspace(input.workspaceId);
if (input.binding.transport !== "rest_api") return input;
return {
...input,
binding: {
...input.binding,
restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
restPath: input.binding.restPath ?? "/health",
},
};
}
backend/src/catalog/types.ts
+28 -1
View File
@@ -2,6 +2,7 @@ export const DATABASE_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"]
export type DatabaseTransport = (typeof DATABASE_TRANSPORTS)[number];
export type ConnectionStatus = "untested" | "reachable" | "failed";
export type CatalogPreprocessingStatus = "running" | "succeeded" | "failed";
export interface DatabaseBinding {
transport: DatabaseTransport;
@@ -36,8 +37,23 @@ export interface WorkspaceDatabase {
lastErrorMessage?: string;
schemaSyncedVersion?: number;
schemaSyncedAt?: string;
metadataContentRevision: number;
preprocessingStatus: CatalogPreprocessingStatus;
preprocessingInputFingerprint?: string;
preprocessedMetadataRevision?: number;
preprocessingStartedAt?: string;
preprocessingFinishedAt?: string;
preprocessingErrorCode?: string;
}
export type CatalogPreprocessingStartResult =
| { kind: "started"; database: WorkspaceDatabase }
| { kind: "not_found" | "schema_stale" | "catalog_busy" | "already_running" };
export type CatalogPreprocessingClearResult =
| { kind: "cleared"; database: WorkspaceDatabase }
| { kind: "not_found" | "already_running" };
export interface CatalogMetrics {
scope: "global" | "database";
databaseId: string | null;
@@ -196,7 +212,7 @@ export interface CatalogLogicalRelationshipCandidate {
export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
export type CatalogDescriptionTarget = "tables" | "columns" | "database_columns";
export type CatalogDescriptionTarget = "tables" | "columns" | "database" | "database_columns";
export interface CatalogMetadataDeleteCounts {
tables: number;
@@ -433,6 +449,17 @@ export interface CatalogRepository {
list(): Promise<WorkspaceDatabase[]>;
get(id: string): Promise<WorkspaceDatabase | undefined>;
getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined>;
beginPreprocessing(
workspaceId: string,
inputFingerprint: string,
): Promise<CatalogPreprocessingStartResult>;
finishPreprocessing(
workspaceId: string,
metadataContentRevision: number,
inputFingerprint: string,
outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
): Promise<WorkspaceDatabase | undefined>;
clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult>;
getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined>;
create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase>;
update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined>;
backend/src/routes/catalog-description-consolidation.ts
+1 -1
View File
@@ -14,7 +14,7 @@ const consolidationSchema = z.discriminatedUnion("target", [
target: z.enum(["tables", "columns"]),
targetIds: z.array(idSchema).min(1).max(10_000),
}).strict(),
z.object({ target: z.literal("database_columns") }).strict(),
z.object({ target: z.enum(["database", "database_columns"]) }).strict(),
]);
function manage(request: FastifyRequest, reply: FastifyReply) {
backend/src/routes/sessions.ts
+63 -8
View File
@@ -11,8 +11,8 @@ import type { WorkspaceRegistry } from "../workspaces/registry.js";
import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js";
import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js";
import { splitCanonicalModelId, type RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
import type { CatalogRepository } from "../catalog/types.js";
const BOOTSTRAP_FAILURE_MESSAGE =
"Session startup failed. Check configuration and connectivity, then Resume the session.";
@@ -42,12 +42,40 @@ export function sessionRoutes(
/** Fail-closed installation/runtime transport capability check. */
workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean;
maintenanceBarrier: MaintenanceBarrier;
/** Optional only for narrow route-test stubs and installations without a Catalog database. */
effectiveRelationships?: EffectiveRelationshipSnapshotProvider;
modelCatalog: RuntimeModelCatalog;
/** PostgreSQL authority for mandatory preprocessing admission. */
catalogRepository?: CatalogRepository;
},
) {
const lifecycleTails = new Map<string, Promise<void>>();
const preprocessingIsCurrent = async (
workspaceId: string,
inputFingerprint?: string,
): Promise<boolean> => {
if (!d.catalogRepository) return true;
const database = await d.catalogRepository.getByWorkspace(workspaceId);
return database !== undefined
&& database.preprocessingStatus === "succeeded"
&& database.preprocessedMetadataRevision === database.metadataContentRevision
&& (inputFingerprint === undefined
|| database.preprocessingInputFingerprint === inputFingerprint);
};
const catalogTransportSupportsSessionRuntime = async (workspaceId: string): Promise<boolean> => {
if (!d.catalogRepository) return true;
const database = await d.catalogRepository.getByWorkspace(workspaceId);
return database === undefined || database.binding.transport !== "ssh_tunnel";
};
const currentInputFingerprint = async (
runner: any,
workspaceConfigPath: string,
): Promise<string | undefined> => (
typeof runner.workspaceInputFingerprint === "function"
? await runner.workspaceInputFingerprint(workspaceConfigPath)
: undefined
);
const boundRuntimes = new Map<
string,
ReturnType<PiProcessManager["createFor"]>
@@ -92,16 +120,13 @@ export function sessionRoutes(
const optionsWithRuntimeConfig = async (
runner: any,
workspaceConfigPath: string | undefined,
workspaceId: string | undefined,
_workspaceId: string | undefined,
options: any,
) => {
if (!workspaceConfigPath || typeof runner.acquireWorkspaceRuntime !== "function") return options;
const effectiveRelationships = workspaceId && d.effectiveRelationships
? await d.effectiveRelationships.render(workspaceId)
: undefined;
return {
...options,
runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath, effectiveRelationships),
runtimeConfig: await runner.acquireWorkspaceRuntime(workspaceConfigPath),
};
};
@@ -379,6 +404,19 @@ export function sessionRoutes(
workspaceId = resolved.revision.id;
workspaceRevision = resolved.revision.commit;
workspaceDescriptor = resolved.workspace;
if (!await catalogTransportSupportsSessionRuntime(workspaceId)) {
return reply.code(409).send({
error: "This workspace transport is not available to runtime sessions.",
code: "workspace_not_activatable",
});
}
const fingerprint = await currentInputFingerprint(runner, workspaceConfigPath);
if (!await preprocessingIsCurrent(workspaceId, fingerprint)) {
return reply.code(409).send({
error: "Run workspace preprocessing before starting the core.",
code: "preprocessing_required",
});
}
} catch {
return reply.code(409).send({
error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE,
@@ -615,6 +653,23 @@ export function sessionRoutes(
workspaceDescriptor = resolved.workspace;
}
catch { return unavailableWorkspaceReply(reply); }
if (d.catalogRepository && saved.workspace_id
&& !await catalogTransportSupportsSessionRuntime(saved.workspace_id)) {
return reply.code(409).send({
error: "This workspace transport is not available to runtime sessions.",
code: "workspace_not_activatable",
});
}
const fingerprint = d.catalogRepository
? await currentInputFingerprint(runner, workspaceConfigPath)
: undefined;
if (d.catalogRepository
&& (!saved.workspace_id || !await preprocessingIsCurrent(saved.workspace_id, fingerprint))) {
return reply.code(409).send({
error: "Run workspace preprocessing before starting the core.",
code: "preprocessing_required",
});
}
try { settings = await d.getSettings(principal); } catch { return storageFailure(reply); }
// This check belongs inside the per-session lock: a preceding cold Resume may have
// installed a running runtime while this request was waiting.
backend/src/routes/workspace-preprocessing.ts
+402
View File
@@ -0,0 +1,402 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import {
CatalogUnavailableError,
type CatalogRepository,
type WorkspaceDatabase,
} from "../catalog/types.js";
import type { ThtRunner } from "../tht/tht-runner.js";
import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
import type { PreprocessingJobState } from "../workspaces/preprocessing-state.js";
import type { WorkspaceRegistry } from "../workspaces/registry.js";
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const ACTIVE_SYNC_STATES = new Set(["queued", "running", "awaiting_confirmation", "applying"]);
export type WorkspacePreprocessingUiState =
| "ready"
| "required"
| "running"
| "blocked"
| "failed";
export interface WorkspacePreprocessingStatus {
schemaVersion: 1;
workspaceId: string;
state: WorkspacePreprocessingUiState;
actionable: boolean;
clearable: boolean;
detail: string;
reason?: string;
nextStep?: string;
metadataRevision?: number;
preprocessedMetadataRevision?: number;
startedAt?: string;
finishedAt?: string;
progress?: {
stage: "catalog_snapshot" | "schema_index" | "evidence" | "finalizing";
step: number;
totalSteps: 4;
};
lastFailure?: {
stage: string;
errorCode: string;
finishedAt: string;
};
}
export interface WorkspacePreprocessingRouteDeps {
repository: CatalogRepository;
registry: WorkspaceRegistry;
service: Pick<WorkspacePreprocessingService, "run" | "clear">;
inputFingerprint?: Pick<ThtRunner, "workspaceInputFingerprint">;
readLatestJob?: (workspaceId: string) => PreprocessingJobState | undefined;
}
const PROGRESS_DETAILS = {
catalog_snapshot: "Preparing the PostgreSQL Catalog snapshot.",
schema_index: "Building schema vectors and LSH indexes.",
evidence: "Indexing Evidence.",
finalizing: "Publishing the completed preprocessing state.",
} as const;
function runningProgress(
workspaceId: string,
deps: WorkspacePreprocessingRouteDeps,
): NonNullable<WorkspacePreprocessingStatus["progress"]> {
let job: PreprocessingJobState | undefined;
try {
job = deps.readLatestJob?.(workspaceId);
} catch {
// Progress is supplemental. A damaged or temporarily unavailable checkpoint must not hide
// the authoritative running state held by PostgreSQL.
}
const completed = new Set(job?.status === "active" ? job.completedStages : []);
if (completed.has("evidence")) return { stage: "finalizing", step: 4, totalSteps: 4 };
if (completed.has("schema_index")) return { stage: "evidence", step: 3, totalSteps: 4 };
if (completed.has("catalog_snapshot")) return { stage: "schema_index", step: 2, totalSteps: 4 };
return { stage: "catalog_snapshot", step: 1, totalSteps: 4 };
}
function base(
workspaceId: string,
state: WorkspacePreprocessingUiState,
detail: string,
database?: WorkspaceDatabase,
): WorkspacePreprocessingStatus {
return {
schemaVersion: 1,
workspaceId,
state,
actionable: state === "ready" || state === "required" || state === "failed",
clearable: Boolean(
database
&& state !== "running"
&& database.preprocessingErrorCode !== "derived_data_cleared"
),
detail,
...(database ? {
metadataRevision: database.metadataContentRevision,
...(database.preprocessedMetadataRevision === undefined
? {}
: { preprocessedMetadataRevision: database.preprocessedMetadataRevision }),
...(database.preprocessingStartedAt ? { startedAt: database.preprocessingStartedAt } : {}),
...(database.preprocessingFinishedAt ? { finishedAt: database.preprocessingFinishedAt } : {}),
} : {}),
};
}
function blocked(
workspaceId: string,
detail: string,
reason: string,
nextStep: string,
database?: WorkspaceDatabase,
): WorkspacePreprocessingStatus {
return { ...base(workspaceId, "blocked", detail, database), reason, nextStep };
}
function failureDiagnostic(errorCode: string): {
stage: string;
detail: string;
reason: string;
nextStep: string;
} {
switch (errorCode) {
case "catalog_snapshot_failed":
return {
stage: "catalog_snapshot",
detail: "The Catalog snapshot could not be prepared.",
reason: "PostgreSQL Catalog metadata could not be read into a consistent preprocessing snapshot.",
nextStep: "Check Catalog availability, then retry preprocessing.",
};
case "schema_index_failed":
return {
stage: "schema_index",
detail: "The schema index could not be rebuilt.",
reason: "The schema indexing worker stopped before the Catalog snapshot was published to Qdrant.",
nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
};
case "evidence_preprocessing_failed":
return {
stage: "evidence",
detail: "Evidence preprocessing did not complete.",
reason: "The Evidence indexing worker stopped before it finished publishing the current workspace data.",
nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
};
case "semantic_index_incompatible":
return {
stage: "semantic_preflight",
detail: "The semantic index configuration is incompatible.",
reason: "Qdrant rejected the collection configuration for the active embedding model.",
nextStep: "Check embedding dimensions and Qdrant collection settings, then retry.",
};
case "egress_policy_refused":
return {
stage: "evidence",
detail: "The Evidence source was refused by policy.",
reason: "The configured Evidence endpoint is not allowed by the installation egress policy.",
nextStep: "Correct the Evidence source or its allowlist configuration, then retry.",
};
case "workspace_not_activatable":
return {
stage: "runtime_preflight",
detail: "The workspace runtime could not be prepared.",
reason: "The active workspace or one of its required runtime bindings is not usable.",
nextStep: "Check Workspace management and Database management, then retry.",
};
default:
return {
stage: "preprocessing",
detail: "Preprocessing did not complete.",
reason: "The preprocessing worker stopped before the current Catalog revision was published.",
nextStep: "Open Last run details below, check the core service log for this error code, then retry.",
};
}
}
export async function readWorkspacePreprocessingStatus(
workspaceId: string,
deps: WorkspacePreprocessingRouteDeps,
): Promise<WorkspacePreprocessingStatus> {
const database = await deps.repository.getByWorkspace(workspaceId);
if (!database) {
return blocked(
workspaceId,
"Database configuration is required.",
"This workspace has no database configuration in the PostgreSQL Catalog.",
"Open Database management and configure the workspace database.",
);
}
if (database.preprocessingStatus === "running") {
const progress = runningProgress(workspaceId, deps);
return {
...base(workspaceId, "running", PROGRESS_DETAILS[progress.stage], database),
progress,
};
}
if (database.binding.transport === "ssh_tunnel") {
return blocked(
workspaceId,
"The database transport is not supported by the core runtime.",
"The current database binding uses an SSH tunnel, which cannot be used by a ThothII session.",
"Open Database management and select a supported runtime transport.",
database,
);
}
if (database.schemaSyncedVersion !== database.version) {
return blocked(
workspaceId,
"Catalog synchronization is required.",
`Database configuration v${database.version} is newer than the latest Catalog synchronization${database.schemaSyncedVersion === undefined ? "." : ` v${database.schemaSyncedVersion}.`}`,
"Open Database management and run Synchronize schema.",
database,
);
}
const [syncRuns, activeDescriptionRun, sensitivityRuns] = await Promise.all([
deps.repository.listSyncRuns(database.id, 10),
deps.repository.getActiveDescriptionGenerationRun(),
deps.repository.listSensitivityAnalysisRuns(50),
]);
if (syncRuns.some((run) => ACTIVE_SYNC_STATES.has(run.state))) {
return blocked(
workspaceId,
"Catalog synchronization is in progress.",
"The Catalog is being synchronized and its metadata revision is not stable yet.",
"Wait for schema synchronization to finish, then run preprocessing.",
database,
);
}
if (activeDescriptionRun?.databaseId === database.id
&& ["queued", "running"].includes(activeDescriptionRun.status)) {
return blocked(
workspaceId,
"Description generation is in progress.",
"Catalog descriptions are still being generated for this database.",
"Wait for description generation to finish, then run preprocessing.",
database,
);
}
if (sensitivityRuns.some((run) => run.databaseId === database.id && run.status === "running")) {
return blocked(
workspaceId,
"Sensitivity analysis is in progress.",
"Catalog sensitivity metadata is still being analyzed for this database.",
"Wait for sensitivity analysis to finish, then run preprocessing.",
database,
);
}
let inputFingerprint: string | undefined;
try {
const record = await deps.registry.read(workspaceId);
if (deps.inputFingerprint) {
inputFingerprint = await deps.inputFingerprint.workspaceInputFingerprint(
record.revision.snapshotPath,
);
}
} catch {
return blocked(
workspaceId,
"The workspace runtime configuration is unavailable.",
"The active workspace revision or one of its required database secrets could not be resolved.",
"Check Workspace management and Database management before running preprocessing.",
database,
);
}
const current = database.preprocessingStatus === "succeeded"
&& database.preprocessedMetadataRevision === database.metadataContentRevision
&& (inputFingerprint === undefined
|| database.preprocessingInputFingerprint === inputFingerprint);
if (current) {
return base(
workspaceId,
"ready",
`Catalog revision ${database.metadataContentRevision} is indexed.`,
database,
);
}
if (database.preprocessingErrorCode === "derived_data_cleared") {
return base(
workspaceId,
"required",
"Reference vectors and LSH are empty. Memory is preserved.",
database,
);
}
if (database.preprocessingStatus === "failed"
&& database.preprocessingErrorCode
&& database.preprocessingErrorCode !== "catalog_changed"
&& database.preprocessingErrorCode !== "derived_data_cleared"
&& database.preprocessingFinishedAt) {
const diagnostic = failureDiagnostic(database.preprocessingErrorCode);
return {
...base(workspaceId, "failed", diagnostic.detail, database),
reason: diagnostic.reason,
nextStep: diagnostic.nextStep,
lastFailure: {
stage: diagnostic.stage,
errorCode: database.preprocessingErrorCode,
finishedAt: database.preprocessingFinishedAt,
},
};
}
return base(
workspaceId,
"required",
`Catalog revision ${database.metadataContentRevision} is not indexed.`,
database,
);
}
function safeError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({
code: "catalog_unavailable",
message: "Database catalog is unavailable.",
});
}
if (error instanceof z.ZodError) {
return reply.code(400).send({
code: "preprocessing_request_invalid",
message: "Preprocessing request is invalid.",
});
}
return reply.code(500).send({
code: "preprocessing_run_failed",
message: "Preprocessing status could not be resolved.",
});
}
function workspaceIdFrom(request: FastifyRequest): string {
return workspaceIdSchema.parse((request.params as { workspaceId?: unknown }).workspaceId);
}
export function workspacePreprocessingRoutes(
app: FastifyInstance,
deps: WorkspacePreprocessingRouteDeps,
): void {
app.get("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply;
try {
return await readWorkspacePreprocessingStatus(workspaceIdFrom(request), deps);
} catch (error) {
return safeError(reply, error);
}
});
app.post("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
try {
const workspaceId = workspaceIdFrom(request);
const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
if (!before.actionable) {
return reply.code(409).send({ ...before, code: "preprocessing_blocked" });
}
const result = await deps.service.run({ workspaceId });
const after = await readWorkspacePreprocessingStatus(workspaceId, deps);
if (after.state === "ready") return after;
if (after.state === "failed") {
return reply.code(422).send({ ...after, code: "preprocessing_run_failed" });
}
if (after.state === "blocked" || after.state === "running") {
return reply.code(409).send({ ...after, code: "preprocessing_blocked" });
}
return reply.code(500).send({
code: "preprocessing_run_failed",
message: `Preprocessing ended with ${result.code}.`,
});
} catch (error) {
return safeError(reply, error);
}
});
app.delete("/workspaces/:workspaceId/preprocessing", async (request, reply) => {
if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
try {
const workspaceId = workspaceIdFrom(request);
const before = await readWorkspacePreprocessingStatus(workspaceId, deps);
if (!before.clearable) {
return reply.code(409).send({ ...before, code: "preprocessing_clear_blocked" });
}
const result = await deps.service.clear({ workspaceId });
if (result.status !== "succeeded") {
return reply.code(result.code === "preprocessing_conflict" ? 409 : 500).send({
code: result.code,
message: "Preprocessing data could not be cleared.",
});
}
return await readWorkspacePreprocessingStatus(workspaceId, deps);
} catch (error) {
return safeError(reply, error);
}
});
}
backend/src/tht/tht-runner.ts
+59 -48
View File
@@ -5,7 +5,7 @@ import {
openSync, readFileSync, readSync, realpathSync, statSync, unlinkSync, writeFileSync,
} from "node:fs";
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
import { parseAllDocuments } from "yaml";
import { parse, parseAllDocuments } from "yaml";
import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js";
import { secretValue, type SecretBundleConfig } from "../config/secret-bundle.js";
import { renderWorkspaceRuntimeFromSnapshotPath } from "../workspaces/runtime-config-lease.js";
@@ -22,6 +22,9 @@ import {
} from "../workspaces/schema.js";
import { reconcileCollection, type CollectionMode } from "../workspaces/qdrant-collection.js";
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
import type { CatalogRepository } from "../catalog/types.js";
import { preprocessingInputFingerprint } from "../workspaces/effective-config.js";
import { workspaceVectorCollections } from "../workspaces/vector-collections.js";
export interface ThtConfig extends SecretBundleConfig {
thtBin: string;
@@ -35,12 +38,14 @@ export interface ThtConfig extends SecretBundleConfig {
/** "self_heal" for session admission (create missing collections/indexes), default "require_existing". */
qdrantCollectionMode?: "self_heal" | "require_existing";
workspaceSecretStore?: WorkspaceSecretStore;
catalogRepository?: CatalogRepository;
}
export interface RuntimeConfigLease {
path: string;
workspaceId: string;
workspaceRevision: string;
inputFingerprint: string;
release(): void;
}
@@ -79,6 +84,7 @@ export type SemanticReadinessCode = "workspace_not_activatable" | "semantic_inde
export interface QdrantEnsureResult {
ok: boolean;
code?: SemanticReadinessCode;
state?: "ready" | "created" | "repaired" | "upgraded";
}
const REQUIRED_QDRANT_PAYLOAD_INDEXES = [
@@ -213,37 +219,31 @@ export class ThtRunner {
}
/** Render one immutable canonical registry revision into a backend-owned harness config. */
acquireWorkspaceRuntime(
workspaceConfigPath: string,
effectiveRelationships?: string,
): RuntimeConfigLease {
const effectiveRelationshipsPath = effectiveRelationships === undefined
async acquireWorkspaceRuntime(workspaceConfigPath: string): Promise<RuntimeConfigLease> {
const identity = this.assertWorkspaceSnapshot(workspaceConfigPath);
const catalogDatabase = this.cfg.catalogRepository === undefined
? undefined
: this.createRuntimeSnapshot(effectiveRelationships);
let rendered: ReturnType<typeof renderWorkspaceRuntimeFromSnapshotPath>;
try {
rendered = renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: workspaceConfigPath,
harnessDir: this.cfg.harnessDir,
configPath: this.cfg.configPath,
dataRoot: this.cfg.dataRoot ?? (() => {
throw new Error("registry workspace runtime requires an absolute data root");
})(),
secretRoots: this.cfg.secretRoots ?? [],
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
workspaceSecretStore: this.cfg.workspaceSecretStore,
effectiveRelationshipsPath,
});
} catch (error) {
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
throw error;
: await this.cfg.catalogRepository.getByWorkspace(identity.workspaceId);
if (this.cfg.catalogRepository !== undefined && !catalogDatabase) {
throw new Error("workspace database is not configured in the Catalog");
}
const rendered = renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: workspaceConfigPath,
harnessDir: this.cfg.harnessDir,
configPath: this.cfg.configPath,
dataRoot: this.cfg.dataRoot ?? (() => {
throw new Error("registry workspace runtime requires an absolute data root");
})(),
secretRoots: this.cfg.secretRoots ?? [],
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
workspaceSecretStore: this.cfg.workspaceSecretStore,
catalogDatabase,
});
let path: string;
try {
path = this.createRuntimeSnapshot(rendered.renderedConfig);
} catch (error) {
rendered.releaseSecrets();
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
throw error;
}
let released = false;
@@ -251,16 +251,29 @@ export class ThtRunner {
path,
workspaceId: rendered.workspaceId,
workspaceRevision: rendered.workspaceRevision,
inputFingerprint: preprocessingInputFingerprint(
rendered.workspaceId,
rendered.workspaceRevision,
parse(rendered.renderedConfig),
),
release: () => {
if (released) return;
released = true;
this.cleanupRuntimeSnapshot(path);
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
rendered.releaseSecrets();
},
};
}
async workspaceInputFingerprint(workspaceConfigPath: string): Promise<string> {
const lease = await this.acquireWorkspaceRuntime(workspaceConfigPath);
try {
return lease.inputFingerprint;
} finally {
lease.release();
}
}
private runtimeSnapshotDirectory(): string {
if (!this.cfg.runtimeSnapshotRoot) throw new Error("runtime snapshot root is not configured");
if (!isAbsolute(this.cfg.runtimeSnapshotRoot)) throw new Error("runtime snapshot root must be absolute");
@@ -392,13 +405,9 @@ export class ThtRunner {
workspaceConfigPath && isAbsolute(workspaceConfigPath)
&& !this.runtimeSnapshots.has(workspaceConfigPath)
) {
let runtime: RuntimeConfigLease;
try {
runtime = this.acquireWorkspaceRuntime(workspaceConfigPath);
} catch (error) {
return Promise.reject(error);
}
return this.run(args, runtime.path, timeoutMs).finally(runtime.release);
return this.acquireWorkspaceRuntime(workspaceConfigPath).then((runtime) => (
this.run(args, runtime.path, timeoutMs).finally(runtime.release)
));
}
return new Promise((resolve) => {
const env: NodeJS.ProcessEnv = { ...process.env };
@@ -598,24 +607,26 @@ export class ThtRunner {
} catch {
return { ok: false, code: "workspace_not_activatable" };
}
const collection = {
collection: descriptor.workspace.id,
dimensions: this.cfg.semanticRuntime.internalEmbeddingDimensions,
distance: "cosine" as const,
};
const collections = workspaceVectorCollections(descriptor.workspace.id);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutSec) * 1000);
try {
const checked = await reconcileCollection({
baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
collection: collection.collection,
dimensions: collection.dimensions,
distance: collection.distance,
mode,
request: this.cfg.qdrantRequest ?? fetch,
signal: controller.signal,
});
return checked;
const checked = await Promise.all(Object.entries(collections).map(async ([purpose, collection]) =>
await reconcileCollection({
baseUrl: this.cfg.semanticRuntime.internalQdrantUrl,
collection,
dimensions: this.cfg.semanticRuntime.internalEmbeddingDimensions,
distance: "cosine",
mode: mode === "evidence_maintenance" && purpose === "memory" ? "self_heal" : mode,
request: this.cfg.qdrantRequest ?? fetch,
signal: controller.signal,
})));
if (!checked.every((result) => result.ok)) {
return { ok: false, code: "semantic_index_incompatible" };
}
const state = (["upgraded", "repaired", "created", "ready"] as const)
.find((candidate) => checked.some((result) => result.state === candidate));
return { ok: true, ...(state ? { state } : {}) };
} catch {
return { ok: false, code: "workspace_not_activatable" };
} finally {
backend/src/workspace-maintenance.ts
+76 -120
View File
@@ -1,15 +1,14 @@
import { spawn } from "node:child_process";
import { closeSync, constants as fsConstants, openSync } from "node:fs";
import { readdir, readFile } from "node:fs/promises";
import { join } from "node:path";
import { parse } from "yaml";
import { loadConfig } from "./config.js";
import { loadConfig, type AppConfig } from "./config.js";
import { ThtRunner } from "./tht/tht-runner.js";
import { WorkspaceRegistry } from "./workspaces/registry.js";
import { publishDeterministicRuntimeConfigLease, renderActiveWorkspaceRuntime } from "./workspaces/runtime-config-lease.js";
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
import { WorkspacePreprocessingService, type WorkspaceOperationResult } from "./workspaces/preprocessing-service.js";
import type { SessionInventoryRow } from "./workspaces/preprocessing-state.js";
import { createCatalogRepository } from "./catalog/repository.js";
import type { CatalogRepository } from "./catalog/types.js";
export interface WorkspaceMaintenanceIo {
stdin: string;
@@ -19,7 +18,7 @@ export interface WorkspaceMaintenanceIo {
writeStderr(value: string): void;
}
type Command = "inspect" | "preprocess-dwh" | "schema-suggest-fks" | "schema-check" | "schema-accept" | "index-schema" | "preprocess-evidence" | "preprocess-run" | "vector-inspect" | "vector-rebuild";
type Command = "inspect" | "preprocess-run" | "preprocess-clear";
function failureResult(
operation: string,
@@ -64,15 +63,8 @@ function parseRequest(command: string, stdin: string): Record<string, unknown> {
}
const allowedByCommand: Record<string, readonly string[]> = {
inspect: ["schemaVersion", "workspaceId"],
"preprocess-dwh": ["schemaVersion", "workspaceId", "resumeRunId"],
"schema-suggest-fks": ["schemaVersion", "workspaceId", "fromSql", "assume", "resumeRunId"],
"schema-check": ["schemaVersion", "workspaceId", "annotationsYaml", "reviewedCandidatesDigest"],
"schema-accept": ["schemaVersion", "workspaceId", "runId", "yes"],
"index-schema": ["schemaVersion", "workspaceId", "resumeRunId"],
"preprocess-evidence": ["schemaVersion", "workspaceId", "dryRun", "resumeRunId"],
"preprocess-run": ["schemaVersion", "workspaceId", "resumeRunId"],
"vector-inspect": ["schemaVersion", "workspaceId"],
"vector-rebuild": ["schemaVersion", "workspaceId", "collection", "confirm", "destroy"],
"preprocess-run": ["schemaVersion", "workspaceId"],
"preprocess-clear": ["schemaVersion", "workspaceId"],
};
const allowed = allowedByCommand[command];
if (!allowed) throw new Error("unknown command");
@@ -91,55 +83,12 @@ async function dispatch(command: Command, service: WorkspacePreprocessingService
switch (command) {
case "inspect":
return await service.inspect({ workspaceId: request.workspaceId as string });
case "preprocess-dwh":
return await service.preprocessDwh({
workspaceId: request.workspaceId as string,
resumeRunId: request.resumeRunId as string | undefined,
});
case "schema-suggest-fks":
return await service.suggestFks({
workspaceId: request.workspaceId as string,
fromSql: request.fromSql as any,
assume: request.assume as any,
resumeRunId: request.resumeRunId as string | undefined,
});
case "schema-check":
return await service.checkSchema({
workspaceId: request.workspaceId as string,
annotationsYaml: request.annotationsYaml as string | undefined,
reviewedCandidatesDigest: request.reviewedCandidatesDigest as string | undefined,
});
case "schema-accept":
return await service.acceptSchema({
workspaceId: request.workspaceId as string,
runId: request.runId as string,
yes: request.yes === true,
});
case "index-schema":
return await service.indexSchema({
workspaceId: request.workspaceId as string,
resumeRunId: request.resumeRunId as string | undefined,
});
case "preprocess-evidence":
return await service.preprocessEvidence({
workspaceId: request.workspaceId as string,
dryRun: request.dryRun as boolean | undefined,
resumeRunId: request.resumeRunId as string | undefined,
});
case "preprocess-run":
return await service.run({
workspaceId: request.workspaceId as string,
resumeRunId: request.resumeRunId as string | undefined,
});
case "vector-inspect":
return await service.vectorInspect({ workspaceId: request.workspaceId as string });
case "vector-rebuild":
return await service.vectorRebuild({
workspaceId: request.workspaceId as string,
collection: request.collection as string | undefined,
confirm: request.confirm as string | undefined,
destroy: request.destroy === true,
});
case "preprocess-clear":
return await service.clear({ workspaceId: request.workspaceId as string });
}
}
@@ -178,48 +127,31 @@ export async function runWorkspaceMaintenanceCli(
|| message === "unexpected request field"
|| message === "invalid workspace id";
return command in {
inspect: true, "preprocess-dwh": true, "schema-suggest-fks": true, "schema-check": true,
"schema-accept": true,
"index-schema": true, "preprocess-evidence": true, "preprocess-run": true,
inspect: true, "preprocess-run": true, "preprocess-clear": true,
} ? (requestError ? 2 : 1) : 2;
}
}
async function readSessionInventory(dataRoot: string, workspaceId: string): Promise<readonly SessionInventoryRow[]> {
const directory = join(dataRoot, "sessions", workspaceId, "sessions");
try {
const entries = await readdir(directory, { withFileTypes: true });
const rows: SessionInventoryRow[] = [];
for (const entry of entries) {
if (!entry.isDirectory() || entry.isSymbolicLink()) continue;
try {
const source = await readFile(join(directory, entry.name, "session_manifest.yaml"), "utf8");
const manifest = parse(source) as Record<string, unknown>;
rows.push({
id: entry.name,
status: typeof manifest.status === "string" ? manifest.status : "open",
archived: manifest.archived === true,
workspaceRevision: typeof manifest.workspace_revision === "string" ? manifest.workspace_revision : null,
});
} catch {
// fail closed at mutation time by ignoring unreadable manifests from the resumable scan
}
}
return rows;
} catch {
return [];
}
export interface ProductionWorkspacePreprocessingDeps {
config?: AppConfig;
catalogRepository?: CatalogRepository;
registry?: WorkspaceRegistry;
workspaceSecretStore?: WorkspaceSecretStore;
runner?: ThtRunner;
}
function createProductionService(): WorkspacePreprocessingService {
const config = loadConfig(process.env, { surface: "workspace-maintenance" });
const registry = new WorkspaceRegistry(config.workspaceRegistry);
const workspaceSecretStore = new WorkspaceSecretStore({
export function createProductionWorkspacePreprocessingService(
deps: ProductionWorkspacePreprocessingDeps = {},
): WorkspacePreprocessingService {
const config = deps.config ?? loadConfig(process.env, { surface: "workspace-maintenance" });
const catalogRepository = deps.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
const registry = deps.registry ?? new WorkspaceRegistry(config.workspaceRegistry);
const workspaceSecretStore = deps.workspaceSecretStore ?? new WorkspaceSecretStore({
root: config.workspaceSecretStoreRoot,
runtimeRoot: config.workspaceSecretRuntimeRoot,
installationId: config.workspaceRegistry.installationId,
});
const runner = new ThtRunner({
const runner = deps.runner ?? new ThtRunner({
thtBin: config.thtBin,
harnessDir: config.harnessDir,
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
@@ -232,16 +164,19 @@ function createProductionService(): WorkspacePreprocessingService {
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
internalEmbeddingId: config.internalEmbeddingId,
internalEmbeddingModel: config.internalEmbeddingModel,
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
},
});
return new WorkspacePreprocessingService({
dataRoot: config.dataRoot ?? "/data",
embeddingDimensions: config.internalEmbeddingDimensions,
httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "")
.split(",").map((value) => value.trim()).filter((value) => value.length > 0),
catalogRepository,
acquireActiveRuntime: async (workspaceId) => {
const catalogDatabase = await catalogRepository.getByWorkspace(workspaceId);
if (!catalogDatabase) throw new Error("workspace database is not configured in the Catalog");
const active = await renderActiveWorkspaceRuntime({
workspaceId,
registry,
@@ -251,6 +186,7 @@ function createProductionService(): WorkspacePreprocessingService {
dataRoot: config.dataRoot ?? "/data",
secretRoots: config.workspaceRegistry.secretRoots,
workspaceSecretStore,
catalogDatabase,
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
@@ -258,38 +194,55 @@ function createProductionService(): WorkspacePreprocessingService {
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
},
});
const configLease = await publishDeterministicRuntimeConfigLease({
workspaceId,
registry,
registryConfig: config.workspaceRegistry,
harnessDir: config.harnessDir,
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
dataRoot: config.dataRoot ?? "/data",
secretRoots: config.workspaceRegistry.secretRoots,
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
internalEmbeddingModel: config.internalEmbeddingModel,
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
},
workspaceSecretStore,
});
return {
workspace: active.workspace,
workspaceId: active.workspaceId,
workspaceRevision: active.workspaceRevision,
descriptorBlob: active.descriptorBlob,
catalogBlob: active.catalogBlob,
configLease,
};
try {
const configLease = await publishDeterministicRuntimeConfigLease({
workspaceId,
registry,
registryConfig: config.workspaceRegistry,
harnessDir: config.harnessDir,
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
dataRoot: config.dataRoot ?? "/data",
secretRoots: config.workspaceRegistry.secretRoots,
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
internalEmbeddingModel: config.internalEmbeddingModel,
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
},
workspaceSecretStore,
catalogDatabase,
});
return {
workspace: active.workspace,
workspaceId: active.workspaceId,
workspaceRevision: active.workspaceRevision,
descriptorBlob: active.descriptorBlob,
catalogBlob: active.catalogBlob,
configLease,
};
} finally {
active.releaseSecrets();
}
},
runChild: async ({ argv, configPath }) => {
const configFd = openSync(configPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
try {
return await new Promise((resolve) => {
const childEnvironment = { ...process.env };
for (const name of [
"THT_CATALOG_DATABASE_URL",
"THT_CATALOG_DB_HOST",
"THT_CATALOG_DB_PORT",
"THT_CATALOG_DB_NAME",
"THT_CATALOG_RUNTIME_USER",
"THT_CATALOG_RUNTIME_PASSWORD_FILE",
"THT_CATALOG_MIGRATOR_DATABASE_URL",
"THT_CATALOG_MIGRATOR_USER",
"THT_CATALOG_MIGRATOR_PASSWORD_FILE",
]) delete childEnvironment[name];
const child = spawn(config.thtBin, argv, {
cwd: config.harnessDir,
env: { ...process.env, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
env: { ...childEnvironment, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
stdio: ["ignore", "pipe", "pipe", configFd],
});
let stdout = "";
@@ -303,9 +256,8 @@ function createProductionService(): WorkspacePreprocessingService {
closeSync(configFd);
}
},
listSessions: async (workspaceId) => await readSessionInventory(config.dataRoot ?? "/data", workspaceId),
semanticPreflight: async (workspace) => {
const result = await runner.qdrantEnsure(workspace, 30);
const result = await runner.qdrantEnsure(workspace, 30, "self_heal");
return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" };
},
evidencePreflight: async (workspace) => {
@@ -330,7 +282,11 @@ if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).
writeStdout: (value) => { stdout.push(value); },
writeStderr: (value) => { stderr.push(value); },
};
const exitCode = await runWorkspaceMaintenanceCli(process.argv, createProductionService(), io);
const exitCode = await runWorkspaceMaintenanceCli(
process.argv,
createProductionWorkspacePreprocessingService(),
io,
);
process.stdout.write(stdout.join(""));
if (stderr.length > 0) process.stderr.write(stderr.join("").slice(0, 64 * 1024));
process.exit(exitCode);
backend/src/workspaces/bindings.ts
+4 -1
View File
@@ -90,6 +90,7 @@ export function resolveBinding(
): ResolvedBinding {
requireSupportedDescriptor(workspace);
const descriptor = validateWorkspaceDescriptor(workspace);
if (!descriptor.dwh) throw new Error("workspace database is not bound in the descriptor");
const contract = buildInstallationContract(descriptor);
const variables = contract.variables.filter((variable) => variable.role === role);
const transportVariable = variables.find((variable) => variable.suffix === "TRANSPORT");
@@ -165,7 +166,9 @@ export function resolveRuntimeBindings(
const descriptor = validateWorkspaceDescriptor(workspace);
return {
dwh: resolveBinding(descriptor, "DWH", env, secretRoots),
dwh: descriptor.dwh
? resolveBinding(descriptor, "DWH", env, secretRoots)
: { transport: "postgres_direct", values: {}, missing: [] },
evidence: resolveEvidenceBinding(descriptor, env, secretRoots),
};
}
backend/src/workspaces/contracts.ts
+3 -1
View File
@@ -155,7 +155,9 @@ export function buildInstallationContract(workspace: WorkspaceDescriptor): Insta
workspaceId: descriptor.workspace.id,
namespace,
variables: [
...connectorVariables(namespace, descriptor.dwh.supported_transports),
...(descriptor.dwh
? connectorVariables(namespace, descriptor.dwh.supported_transports)
: []),
...evidenceVariables(namespace, descriptor),
],
};
backend/src/workspaces/diagnostics.ts
+16 -14
View File
@@ -12,6 +12,7 @@ import {
import type { WorkspaceErrorCode } from "./types.js";
import type { SemanticRuntimeConfig } from "./runtime-renderer.js";
import type { AuthDiagnostics } from "../auth/diagnostics.js";
import { workspaceVectorCollections } from "./vector-collections.js";
export interface Diagnostic {
level: "error" | "warning" | "info";
@@ -447,6 +448,9 @@ async function diagnoseValidatedWorkspace(
let activatable = true;
if (!skipDwh) {
if (!descriptor.dwh) {
return { activatable: false, diagnostics: [diagnosticError("binding_missing", "dwh")] };
}
const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs);
const dwhValues = bindings.dwh.values;
const dwhField = (suffix: string) => bindingName(descriptor, suffix);
@@ -514,20 +518,18 @@ async function diagnoseValidatedWorkspace(
}
try {
const vector = await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
baseUrl: semanticRuntime.internalQdrantUrl,
collection: descriptor.workspace.id,
timeoutMs,
signal,
}));
const expected = {
collection: descriptor.workspace.id,
dimensions: semanticRuntime.internalEmbeddingDimensions,
distance: "cosine",
};
if (vector.collection !== expected.collection
|| vector.dimensions !== expected.dimensions
|| vector.distance !== expected.distance) {
const expectedCollections = Object.values(workspaceVectorCollections(descriptor.workspace.id));
const vectors = await Promise.all(expectedCollections.map(async (collection) =>
await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
baseUrl: semanticRuntime.internalQdrantUrl,
collection,
timeoutMs,
signal,
}))));
if (vectors.some((vector, index) =>
vector.collection !== expectedCollections[index]
|| vector.dimensions !== semanticRuntime.internalEmbeddingDimensions
|| vector.distance !== "cosine")) {
diagnostics.push(diagnosticError("semantic_index_incompatible"));
activatable = false;
}
backend/src/workspaces/effective-config.ts
+35 -5
View File
@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
import { normalize } from "node:path";
export interface CanonicalEffectiveConfig {
schemaVersion: 2;
schemaVersion: 3;
dwh: CanonicalDwhConfig;
vector: CanonicalVectorConfig;
embedding: CanonicalEmbeddingConfig;
@@ -21,7 +21,10 @@ export interface CanonicalDwhConfig {
}
export interface CanonicalVectorConfig {
collection: string;
collections: {
reference: string;
memory: string;
};
dimensions: number;
distance: string;
}
@@ -120,7 +123,10 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
if (!vector) {
throw new TypeError("effective config is missing vector resources");
}
const collection = requireString(vector, "collection");
const collections = asRecord(vector.collections);
if (!collections) {
throw new TypeError("effective config is missing vector collections");
}
const semanticIndex = asRecord(rendered.semantic_index);
const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined;
const dimensions = vectorStore
@@ -129,7 +135,14 @@ function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorCo
const distance = vectorStore
? requireString(vectorStore, "distance")
: (optionalString(vector, "distance") ?? "cosine");
return { collection, dimensions, distance };
return {
collections: {
reference: requireString(collections, "reference"),
memory: requireString(collections, "memory"),
},
dimensions,
distance,
};
}
function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbeddingConfig {
@@ -169,7 +182,7 @@ export function buildCanonicalEffectiveConfig(renderedConfig: unknown): Canonica
throw new TypeError("effective config requires a rendered configuration object");
}
return {
schemaVersion: 2,
schemaVersion: 3,
dwh: buildDwhConfig(rendered),
vector: buildVectorConfig(rendered),
embedding: buildEmbeddingConfig(rendered),
@@ -220,3 +233,20 @@ export function configFingerprint(renderedConfig: unknown): string {
export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string {
return sha256(effectiveConfigIdentity(workspaceId, renderedConfig));
}
/**
* Fingerprint every non-Catalog input consumed by complete preprocessing. The immutable Git
* revision covers the workspace descriptor and its revision-pinned Evidence tree; the effective
* configuration identity covers the Catalog-derived DWH binding and semantic runtime contract.
*/
export function preprocessingInputFingerprint(
workspaceId: string,
workspaceRevision: string,
renderedConfig: unknown,
): string {
return sha256(JSON.stringify({
workspaceId,
workspaceRevision,
effectiveConfigIdentity: effectiveConfigIdentity(workspaceId, renderedConfig),
}));
}
backend/src/workspaces/evidence/preprocessing.ts
+8
View File
@@ -171,6 +171,14 @@ export async function continueEvidencePreprocessing(
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
if (policy) return { ...policy, ...jobResult(request.job) };
if (!request.job.completedStages.includes("evidence")) {
const preflight = await deps.evidencePreflight();
if (!preflight.ok) {
return {
status: "failed",
code: preflight.code,
...jobResult(request.job),
};
}
return await runEvidenceStage(request, deps);
}
return { status: "unchanged", code: "ok", ...jobResult(request.job) };
backend/src/workspaces/preprocessing-service.ts
+156 -342
View File
@@ -1,22 +1,19 @@
import { createHash, randomBytes } from "node:crypto";
import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
import { randomBytes } from "node:crypto";
import { renameSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import {
continueEvidencePreprocessing,
preprocessEvidence as runEvidencePreprocessing,
type EvidencePreprocessingDependencies,
type EvidencePreprocessingOutcome,
} from "./evidence/preprocessing.js";
import type { WorkspaceDescriptor } from "./schema.js";
import {
PreprocessingStateStore,
type FkReviewRecord,
type PreprocessingJobState,
type SessionInventoryRow,
} from "./preprocessing-state.js";
import type { DeterministicRuntimeConfigLease } from "./runtime-config-lease.js";
import { readAnnotationsSync } from "./annotations-sync.js";
import { reconcileCollection } from "./qdrant-collection.js";
import { buildCatalogMetadataSnapshot } from "../catalog/metadata-snapshot.js";
import type { CatalogRepository } from "../catalog/types.js";
export interface WorkspaceOperationResult {
schemaVersion: 1;
@@ -27,7 +24,7 @@ export interface WorkspaceOperationResult {
| "preprocessing_resume_mismatch" | "manual_review_required"
| "evidence_materialization_required" | "effective_config_mismatch"
| "semantic_index_incompatible" | "annotation_invalid"
| "egress_policy_refused";
| "egress_policy_refused" | "catalog_not_ready" | "preprocessing_clear_failed";
workspaceId: string;
workspaceRevision: string;
descriptorBlob: string;
@@ -69,7 +66,6 @@ export interface WorkspacePreprocessingServiceDeps {
dataRoot: string;
acquireActiveRuntime(workspaceId: string): Promise<ActiveRuntime>;
runChild(request: ChildProcessRequest): Promise<ChildProcessResult>;
listSessions(workspaceId: string): Promise<readonly SessionInventoryRow[]>;
semanticPreflight(workspace: WorkspaceDescriptor): Promise<
{ ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
>;
@@ -77,7 +73,7 @@ export interface WorkspacePreprocessingServiceDeps {
{ ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
>;
httpPrivateHostAllowlist?: readonly string[];
embeddingDimensions?: number;
catalogRepository?: CatalogRepository;
}
interface RunScope {
@@ -86,10 +82,6 @@ interface RunScope {
job: PreprocessingJobState;
}
function digest(value: string | Buffer): string {
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
}
function baseResult(
runtime: ActiveRuntime,
operation: string,
@@ -116,41 +108,6 @@ function baseResult(
export class WorkspacePreprocessingService {
constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
async vectorInspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
const collection = runtime.workspace.workspace.id;
const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" });
if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] });
const body = await res.json() as any;
const info = body?.result;
const vectors = info?.config?.params?.vectors;
return baseResult(runtime, "vector inspect", "succeeded", "ok", {
counts: { dimensions: vectors?.size ?? 0 },
warnings: [`collection=${collection} distance=${vectors?.distance ?? "unknown"}`],
});
}
async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise<WorkspaceOperationResult> {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
const collection = runtime.workspace.workspace.id;
if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) {
return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] });
}
const q = `${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`;
const del = await fetch(q, { method: "DELETE" });
if (!del.ok && del.status !== 404) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection delete failed"] });
// Recreate the complete contract (dimensions + distance + the 8 required keyword indexes).
const recreated = await reconcileCollection({
baseUrl: runtime.configLease.semanticQdrantUrl,
collection,
dimensions: this.deps.embeddingDimensions ?? 1024,
distance: "cosine",
mode: "self_heal",
});
if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] });
return baseResult(runtime, "vector rebuild", "succeeded", "ok", { warnings: [`recreated collection=${collection}`] });
}
async inspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
try {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
@@ -175,227 +132,151 @@ export class WorkspacePreprocessingService {
}
}
async preprocessDwh(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
const scope = await this.startRun(options.workspaceId, "preprocess dwh", options.resumeRunId);
if (scope.job.completedStages.includes("dwh")) {
return baseResult(scope.runtime, "preprocess dwh", "unchanged", "ok", {
runId: scope.job.runId,
childRuns: scope.job.childRuns,
completedStages: [...scope.job.completedStages],
});
async run(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
if (!this.deps.catalogRepository) {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
return baseResult(runtime, "preprocess run", "failed", "catalog_not_ready");
}
const payload = await this.runJsonStage(scope.runtime, [
"preprocess", "dwh", "--steps", "introspect,lsh",
...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
"--json", "-c", "/dev/fd/3",
]);
const childRun = this.requireRunId(payload.run_id);
scope.job.childRuns.dwh = childRun;
if (!scope.job.completedStages.includes("dwh")) scope.job.completedStages.push("dwh");
this.state(scope.runtime.workspaceId).writeJob(scope.job);
return baseResult(scope.runtime, "preprocess dwh", "succeeded", "ok", {
runId: scope.job.runId,
childRuns: { ...scope.job.childRuns },
completedStages: [...scope.job.completedStages],
});
return await this.runFromCatalog(options);
}
async suggestFks(options: {
workspaceId: string;
fromSql?: ReadonlyArray<{ name: string; sql: string }>;
assume?: readonly string[];
resumeRunId?: string;
}): Promise<WorkspaceOperationResult> {
const scope = await this.startRun(options.workspaceId, "schema suggest-fks", options.resumeRunId);
return await this.runSuggestStage(scope, options.fromSql ?? [], options.assume ?? []);
}
async checkSchema(options: {
workspaceId: string;
annotationsYaml?: string;
reviewedCandidatesDigest?: string;
}): Promise<WorkspaceOperationResult> {
async clear(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
const state = this.state(runtime.workspaceId);
if ((options.annotationsYaml === undefined) !== (options.reviewedCandidatesDigest === undefined)) {
return baseResult(runtime, "schema check", "failed", "annotation_invalid");
const repository = this.deps.catalogRepository;
if (!repository) return baseResult(runtime, "preprocess clear", "failed", "catalog_not_ready");
const cleared = await repository.clearPreprocessing(runtime.workspaceId);
if (cleared.kind !== "cleared") {
return baseResult(
runtime,
"preprocess clear",
"failed",
cleared.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
);
}
if (options.reviewedCandidatesDigest === undefined) {
const payload = await this.runJsonStage(runtime, ["schema", "check", "--json", "-c", "/dev/fd/3"]);
return baseResult(runtime, "schema check", Number(payload.orphan_count ?? 0) === 0 ? "succeeded" : "failed", Number(payload.orphan_count ?? 0) === 0 ? "ok" : "annotation_invalid");
}
const reviewedCandidatesDigest = options.reviewedCandidatesDigest;
const runId = this.findRunIdByCandidateDigest(state, reviewedCandidatesDigest);
if (!runId) return baseResult(runtime, "schema check", "failed", "annotation_invalid");
const request = await this.withStagedInputs(runtime.workspaceId, [
{ flag: "--annotations", name: "annotations.yaml", contents: options.annotationsYaml! },
], async (argv) => await this.runJsonStage(runtime, [
"schema", "check", ...argv,
"--reviewed-candidates", reviewedCandidatesDigest,
"--json", "-c", "/dev/fd/3",
]));
if (request.reviewed_candidates_digest !== reviewedCandidatesDigest || typeof request.annotations_digest !== "string") {
return baseResult(runtime, "schema check", "failed", "annotation_invalid", { runId });
}
// P5 supersedes the host-file FK review: schema check is read-only validation and never
// records a review. Only `schema accept` records a human review for the curated Git blob.
return baseResult(runtime, "schema check", "succeeded", "ok", { runId });
}
async acceptSchema(options: { workspaceId: string; runId: string; yes?: boolean }): Promise<WorkspaceOperationResult> {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
const state = this.state(runtime.workspaceId);
if (options.yes !== true) {
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
runId: options.runId,
warnings: ["accept requires --yes"],
});
}
if (!/^[0-9a-f]{32}$/.test(options.runId)) {
return baseResult(runtime, "schema accept", "failed", "annotation_invalid");
}
const candidate = state.readFkCandidates(options.runId);
if (candidate === undefined) {
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
runId: options.runId,
warnings: ["candidate run is unavailable"],
});
}
const synced = readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision);
if (synced === undefined || synced.contents.toString("utf8").trim() === "") {
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
runId: options.runId,
warnings: ["curated annotations are not synchronized"],
});
}
// The harness parser validates the curated blob against the physical schema; the recorded
// candidate digest must round-trip and the blob digest must match the synced destination.
const payload = await this.runJsonStage(runtime, [
"schema", "check", "--reviewed-candidates", candidate.digest, "--json", "-c", "/dev/fd/3",
]);
if (payload.annotations_digest !== synced.contentDigest
|| payload.reviewed_candidates_digest !== candidate.digest
|| Number(payload.orphan_count ?? 0) !== 0) {
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { runId: options.runId });
}
const review = state.writeFkReview(options.runId, {
reviewedCandidatesDigest: candidate.digest,
annotationsDigest: synced.contentDigest,
workspaceRevision: runtime.workspaceRevision,
blobId: synced.blobId,
const result = await this.deps.runChild({
argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
configPath: runtime.configLease.path,
});
const job = state.readJob(options.runId);
job.reviewDigest = review.digest;
if (!job.completedStages.includes("fk_review")) job.completedStages.push("fk_review");
state.writeJob(job);
return baseResult(runtime, "schema accept", "succeeded", "ok", {
runId: options.runId,
completedStages: [...job.completedStages],
artifactIdentities: [
{ kind: "fk_review", digest: review.digest },
{ kind: "annotations", digest: synced.contentDigest },
],
if (result.exitCode !== 0) {
return baseResult(runtime, "preprocess clear", "failed", "preprocessing_clear_failed");
}
const payload = JSON.parse(result.stdout) as Record<string, unknown>;
return baseResult(runtime, "preprocess clear", "succeeded", "ok", {
counts: this.numberRecord(payload.counts),
});
}
async indexSchema(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
const scope = await this.startRun(options.workspaceId, "index-schema", options.resumeRunId);
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
if (!semantic.ok) return baseResult(scope.runtime, "index-schema", "failed", semantic.code, { runId: scope.job.runId });
if (scope.job.completedStages.includes("schema_index")) {
return baseResult(scope.runtime, "index-schema", "unchanged", "ok", {
runId: scope.job.runId,
completedStages: [...scope.job.completedStages],
});
private async runFromCatalog(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
const scope = await this.startRun(options.workspaceId);
const repository = this.deps.catalogRepository!;
const fingerprint = scope.runtime.configLease.inputFingerprint;
const started = await repository.beginPreprocessing(scope.runtime.workspaceId, fingerprint);
if (started.kind !== "started") {
scope.job.status = "failed";
scope.state.writeJob(scope.job);
return baseResult(
scope.runtime,
"preprocess run",
"failed",
started.kind === "already_running" ? "preprocessing_conflict" : "catalog_not_ready",
{ warnings: [`catalog=${started.kind}`] },
);
}
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
const counts = this.numberRecord(payload.counts);
scope.job.completedStages.push("schema_index");
this.state(scope.runtime.workspaceId).writeJob(scope.job);
return baseResult(scope.runtime, "index-schema", "succeeded", "ok", {
runId: scope.job.runId,
completedStages: [...scope.job.completedStages],
counts,
});
}
async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId);
const outcome = await runEvidencePreprocessing(
{
evidence: scope.runtime.workspace.evidence,
job: scope.job,
dryRun: options.dryRun,
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
},
this.evidenceDependencies(scope),
);
return this.evidenceResult(scope, "preprocess evidence", outcome);
}
async run(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
const scope = await this.startRun(options.workspaceId, "preprocess run", options.resumeRunId);
if (!scope.job.completedStages.includes("dwh")) {
const payload = await this.runJsonStage(scope.runtime, [
"preprocess", "dwh", "--steps", "introspect,lsh",
...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
"--json", "-c", "/dev/fd/3",
]);
scope.job.childRuns.dwh = this.requireRunId(payload.run_id);
scope.job.completedStages.push("dwh");
this.state(scope.runtime.workspaceId).writeJob(scope.job);
}
if (!scope.job.completedStages.includes("fk_suggest")) {
const suggest = await this.runSuggestStage(scope, [], []);
if (suggest.code === "manual_review_required") return suggest;
}
const candidate = this.state(scope.runtime.workspaceId).readFkCandidates(scope.job.runId);
if (candidate && !scope.job.completedStages.includes("fk_review")) {
// P5: continuation requires a review accepted for this candidate whose accepted blob digest
// equals the current revision's synced annotations. A revision change (or a missing curated
// blob) therefore records a new review checkpoint instead of silently reusing the old one.
const accepted = this.findAcceptedReviewForDigest(scope.runtime.workspaceId, candidate.digest);
const currentDigest = this.currentAnnotationsDigest(scope.runtime);
if (accepted === undefined || currentDigest === undefined || accepted.annotationsDigest !== currentDigest) {
return baseResult(scope.runtime, "preprocess run", "blocked", "manual_review_required", {
runId: scope.job.runId,
childRuns: { ...scope.job.childRuns },
completedStages: [...scope.job.completedStages],
artifactIdentities: [{ kind: "fk_candidates", digest: candidate.digest }],
});
const revision = started.database.metadataContentRevision;
let finished = false;
let failureCode = "catalog_snapshot_failed";
try {
const snapshot = await buildCatalogMetadataSnapshot(
repository,
scope.runtime.workspaceId,
revision,
);
const snapshotPath = this.publishCatalogSnapshot(
scope.runtime.workspaceId,
JSON.stringify(snapshot),
);
this.completeStages(scope, "catalog_snapshot");
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
if (!semantic.ok) {
await repository.finishPreprocessing(
scope.runtime.workspaceId,
revision,
fingerprint,
{ status: "failed", errorCode: semantic.code },
);
scope.job.status = "failed";
scope.state.writeJob(scope.job);
finished = true;
return baseResult(scope.runtime, "preprocess run", "failed", semantic.code);
}
scope.job.reviewDigest = accepted.reviewedCandidatesDigest;
scope.job.completedStages.push("fk_review");
this.state(scope.runtime.workspaceId).writeJob(scope.job);
failureCode = "schema_index_failed";
const payload = await this.runJsonStage(scope.runtime, [
"preprocess",
"catalog",
"--catalog-metadata",
snapshotPath,
"--json",
"-c",
"/dev/fd/3",
]);
this.completeStages(scope, "catalog_metadata", "lsh", "schema_index");
failureCode = "evidence_preprocessing_failed";
const outcome = await continueEvidencePreprocessing(
{
evidence: scope.runtime.workspace.evidence,
job: scope.job,
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
priorCounts: this.numberRecord(payload.counts),
},
this.evidenceDependencies(scope),
);
if (!["succeeded", "unchanged"].includes(outcome.status)) {
await repository.finishPreprocessing(
scope.runtime.workspaceId,
revision,
fingerprint,
{ status: "failed", errorCode: outcome.code },
);
scope.job.status = "failed";
scope.state.writeJob(scope.job);
finished = true;
return this.evidenceResult(scope, outcome);
}
// Evidence has been published. The only remaining operation is the atomic Catalog commit.
this.completeStages(scope, "evidence");
const completed = await repository.finishPreprocessing(
scope.runtime.workspaceId,
revision,
fingerprint,
{ status: "succeeded" },
);
if (!completed) throw new Error("catalog preprocessing completion lost its lease");
scope.job.status = "succeeded";
scope.state.writeJob(scope.job);
finished = true;
return this.evidenceResult(scope, outcome);
} catch (error) {
if (!finished) {
await repository.finishPreprocessing(
scope.runtime.workspaceId,
revision,
fingerprint,
{ status: "failed", errorCode: failureCode },
);
}
scope.job.status = "failed";
scope.state.writeJob(scope.job);
throw error;
}
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId });
let schemaCounts: Record<string, number> | undefined;
if (!scope.job.completedStages.includes("schema_index")) {
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
scope.job.completedStages.push("schema_index");
this.state(scope.runtime.workspaceId).writeJob(scope.job);
schemaCounts = this.numberRecord(payload.counts);
}
const outcome = await continueEvidencePreprocessing(
{
evidence: scope.runtime.workspace.evidence,
job: scope.job,
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
priorCounts: schemaCounts,
},
this.evidenceDependencies(scope),
);
return this.evidenceResult(scope, "preprocess run", outcome);
}
private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise<RunScope> {
private async startRun(workspaceId: string): Promise<RunScope> {
const runtime = await this.deps.acquireActiveRuntime(workspaceId);
const state = this.state(runtime.workspaceId);
await state.assertSessionInventoryCompatible(runtime.workspaceRevision, await this.deps.listSessions(runtime.workspaceId));
const job = await state.beginJob({
operation,
runId: resumeRunId,
operation: "preprocess run",
workspaceRevision: runtime.workspaceRevision,
descriptorBlob: runtime.descriptorBlob,
catalogBlob: runtime.catalogBlob,
@@ -411,6 +292,28 @@ export class WorkspacePreprocessingService {
return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId });
}
private completeStages(scope: RunScope, ...stages: string[]): void {
for (const stage of stages) {
if (!scope.job.completedStages.includes(stage)) scope.job.completedStages.push(stage);
}
scope.state.writeJob(scope.job);
}
private publishCatalogSnapshot(workspaceId: string, contents: string): string {
const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing");
mkdirSync(root, { recursive: true, mode: 0o700 });
const target = join(root, "catalog-metadata.json");
const staging = join(root, `.catalog-metadata-${randomBytes(6).toString("hex")}.json`);
try {
writeFileSync(staging, contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
renameSync(staging, target);
return target;
} catch (error) {
rmSync(staging, { force: true });
throw error;
}
}
private evidenceDependencies(scope: RunScope): EvidencePreprocessingDependencies {
return {
runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
@@ -423,50 +326,10 @@ export class WorkspacePreprocessingService {
private evidenceResult(
scope: RunScope,
operation: "preprocess evidence" | "preprocess run",
outcome: EvidencePreprocessingOutcome,
): WorkspaceOperationResult {
const { status, code, ...extra } = outcome;
return baseResult(scope.runtime, operation, status, code, extra);
}
private async runSuggestStage(
scope: RunScope,
fromSql: ReadonlyArray<{ name: string; sql: string }>,
assume: readonly string[],
): Promise<WorkspaceOperationResult> {
const payload = await this.withStagedInputs(scope.runtime.workspaceId, fromSql.map((entry) => ({
flag: "--from-sql",
name: entry.name,
contents: entry.sql,
})), async (stagedArgv) => await this.runJsonStage(scope.runtime, [
"schema", "suggest-fks", ...stagedArgv,
...assume.flatMap((value) => ["--assume", value]),
"--json", "-c", "/dev/fd/3",
]));
const candidateCount = Number(payload.candidate_count ?? 0);
const candidateYaml = typeof payload.candidate_yaml === "string" ? payload.candidate_yaml : "";
let artifactIdentities: Array<{ kind: string; digest: string }> | undefined;
if (candidateCount > 0) {
const persisted = this.state(scope.runtime.workspaceId).writeFkCandidates(scope.job.runId, candidateYaml);
scope.job.candidateDigest = persisted.digest;
artifactIdentities = [{ kind: "fk_candidates", digest: persisted.digest }];
}
if (!scope.job.completedStages.includes("fk_suggest")) scope.job.completedStages.push("fk_suggest");
this.state(scope.runtime.workspaceId).writeJob(scope.job);
const resultExtra = {
runId: scope.job.runId,
completedStages: [...scope.job.completedStages],
...(artifactIdentities ? { artifactIdentities } : {}),
...(candidateYaml.length > 0 ? { suggestedFksYaml: candidateYaml } : {}),
};
if (candidateCount > 0) {
return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "blocked", "manual_review_required", {
...resultExtra,
childRuns: { ...scope.job.childRuns },
});
}
return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "succeeded", "ok", resultExtra);
return baseResult(scope.runtime, "preprocess run", status, code, extra);
}
private async runJsonStage(runtime: ActiveRuntime, argv: string[]): Promise<Record<string, unknown>> {
@@ -485,54 +348,5 @@ export class WorkspacePreprocessingService {
return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([key, nested]) => [key, Number(nested)]));
}
private async withStagedInputs<T>(
workspaceId: string,
inputs: ReadonlyArray<{ flag: string; name: string; contents: string }>,
fn: (argv: string[]) => Promise<T>,
): Promise<T> {
if (inputs.length === 0) return await fn([]);
const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing", `.stage-${randomBytes(6).toString("hex")}`);
mkdirSync(root, { recursive: true, mode: 0o700 });
const argv: string[] = [];
const paths: string[] = [];
try {
for (const input of inputs) {
const path = join(root, input.name);
writeFileSync(path, input.contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
paths.push(path);
argv.push(input.flag, path);
}
return await fn(argv);
} finally {
rmSync(root, { recursive: true, force: true });
}
}
private currentAnnotationsDigest(runtime: ActiveRuntime): string | undefined {
return readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision)?.contentDigest;
}
private findAcceptedReviewForDigest(
workspaceId: string,
digestValue: string,
): FkReviewRecord | undefined {
const state = this.state(workspaceId);
for (const entry of readdirSync(state.fkReviewsDirectory(), { withFileTypes: true })) {
if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.json$/.test(entry.name)) continue;
const runId = entry.name.slice(0, -".json".length);
const review = state.readFkReview(runId);
if (review && review.reviewedCandidatesDigest === digestValue) return review;
}
return undefined;
}
private findRunIdByCandidateDigest(state: PreprocessingStateStore, digestValue: string): string | undefined {
for (const entry of readdirSync(state.fkCandidatesDirectory(), { withFileTypes: true })) {
if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.yaml$/.test(entry.name)) continue;
const runId = entry.name.slice(0, -".yaml".length);
if (state.readFkCandidates(runId)?.digest === digestValue) return runId;
}
return undefined;
}
}
backend/src/workspaces/preprocessing-state.ts
+18 -6
View File
@@ -198,6 +198,7 @@ export class PreprocessingStateStore {
runtimeConfigDirectory(): string { return join(this.root, "runtime-config"); }
runtimeConfigManifestDirectory(): string { return join(this.root, "runtime-config-manifests"); }
jobsDirectory(): string { return join(this.root, "jobs"); }
latestJobPath(): string { return join(this.root, "latest-job.json"); }
fkCandidatesDirectory(): string { return join(this.root, "fk-candidates"); }
fkReviewsDirectory(): string { return join(this.root, "fk-reviews"); }
jobPath(runId: string): string { return join(this.jobsDirectory(), `${validateRunId(runId)}.json`); }
@@ -289,7 +290,7 @@ export class PreprocessingStateStore {
"Workspace preprocessing resume no longer matches the pinned revision",
);
}
return existing;
return this.writeJob(existing);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
if (error instanceof PreprocessingStateError) throw error;
@@ -318,19 +319,30 @@ export class PreprocessingStateStore {
childRuns: {},
status: "active",
};
writeAtomicFile(path, `${JSON.stringify(job)}
`, 0o600);
return job;
return this.writeJob(job);
}
readJob(runId: string): PreprocessingJobState {
return decodeJob(JSON.parse(readTrustedFile(this.jobPath(runId))));
}
readLatestJob(): PreprocessingJobState | undefined {
try {
return decodeJob(JSON.parse(readTrustedFile(this.latestJobPath())));
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
throw error;
}
}
writeJob(job: PreprocessingJobState): PreprocessingJobState {
this.ensureLayout();
writeAtomicFile(this.jobPath(job.runId), `${JSON.stringify(job)}
`, 0o600);
const contents = `${JSON.stringify(job)}
`;
writeAtomicFile(this.jobPath(job.runId), contents, 0o600);
// This fixed pointer is the sole status surface for operators. Per-run files remain an
// internal resume mechanism and are never exposed as history.
writeAtomicFile(this.latestJobPath(), contents, 0o600);
return job;
}
backend/src/workspaces/runtime-config-lease.ts
+40 -31
View File
@@ -23,7 +23,7 @@ import {
canonicalEffectiveConfigJson,
configFingerprint,
effectiveConfigIdentity,
inputFingerprint,
preprocessingInputFingerprint,
type CanonicalEffectiveConfig,
} from "./effective-config.js";
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
@@ -41,6 +41,8 @@ import {
} from "./runtime-renderer.js";
import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js";
import type { WorkspaceRegistryConfig } from "./types.js";
import { resolveCatalogRuntimeBinding } from "../catalog/runtime-binding.js";
import type { WorkspaceDatabase } from "../catalog/types.js";
export interface RuntimeConfigLease {
path: string;
@@ -246,8 +248,6 @@ function readSnapshotWorkspace(snapshotPath: string): {
function runtimePaths(
dataRoot: string,
workspaceId: string,
workspaceRevision?: string,
effectiveRelationshipsPath?: string,
): RuntimePaths {
if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
const root = join(dataRoot, "sessions", workspaceId);
@@ -256,12 +256,7 @@ function runtimePaths(
artifacts: join(root, "artifacts"),
indexes: join(root, "indexes"),
memory: join(root, "memory"),
...(workspaceRevision === undefined
? {}
: { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }),
...(effectiveRelationshipsPath === undefined
? {}
: { effective_relationships: effectiveRelationshipsPath }),
catalog_metadata_snapshot: join(root, "preprocessing", "catalog-metadata.json"),
};
}
@@ -309,19 +304,32 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
effectiveRelationshipsPath?: string;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): RenderedWorkspaceRuntime {
const secretLease = options.workspaceSecretStore === undefined
if (options.catalogDatabase && !options.workspaceSecretStore) {
throw new Error("Catalog runtime binding requires the workspace secret store");
}
const catalogLease = options.catalogDatabase === undefined
? undefined
: resolveCatalogRuntimeBinding({
workspace: options.workspace,
database: options.catalogDatabase,
environment: process.env,
secretRoots: options.secretRoots,
secretStore: options.workspaceSecretStore!,
});
const runtimeWorkspace = catalogLease?.workspace ?? options.workspace;
const secretLease = catalogLease !== undefined || options.workspaceSecretStore === undefined
? undefined
: resolveRuntimeBindingsWithWorkspaceSecrets(
options.workspace,
runtimeWorkspace,
process.env,
options.secretRoots,
options.workspaceSecretStore,
);
const bindings = secretLease?.bindings
?? resolveRuntimeBindings(options.workspace, process.env, options.secretRoots);
const bindings = catalogLease?.bindings ?? secretLease?.bindings
?? resolveRuntimeBindings(runtimeWorkspace, process.env, options.secretRoots);
const overlay = installationOverlay(options.harnessDir, options.configPath);
const context: RuntimeRenderContext = {
workspaceId: options.workspaceId,
@@ -334,32 +342,26 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
runtimePaths: runtimePaths(
options.dataRoot,
options.workspaceId,
options.workspaceRevision,
options.effectiveRelationshipsPath,
),
runtimePaths: runtimePaths(options.dataRoot, options.workspaceId),
installationOverlay: overlay,
bindings,
bindingDigest: stableBindingDigest(bindings),
semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
releaseSecrets: () => secretLease?.release(),
releaseSecrets: () => {
catalogLease?.release();
secretLease?.release();
},
renderedConfig: renderRuntimeConfig(
options.workspace,
runtimeWorkspace,
bindings,
runtimePaths(
options.dataRoot,
options.workspaceId,
options.workspaceRevision,
options.effectiveRelationshipsPath,
),
runtimePaths(options.dataRoot, options.workspaceId),
context,
overlay,
options.semanticRuntime,
),
};
} catch (error) {
catalogLease?.release();
secretLease?.release();
throw error;
}
@@ -372,8 +374,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
effectiveRelationshipsPath?: string;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): RenderedWorkspaceRuntime {
const snapshot = readSnapshotWorkspace(options.snapshotPath);
return renderWorkspaceRuntimeFromWorkspace({
@@ -386,8 +388,8 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: options.dataRoot,
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
effectiveRelationshipsPath: options.effectiveRelationshipsPath,
workspaceSecretStore: options.workspaceSecretStore,
catalogDatabase: options.catalogDatabase,
});
}
@@ -401,6 +403,7 @@ export async function renderActiveWorkspaceRuntime(options: {
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): Promise<ActiveRenderedWorkspaceRuntime> {
// The persisted active state may reference host-side snapshot paths (written by another
// process or installation). Read the active state directly and resolve the immutable snapshot
@@ -431,6 +434,7 @@ export async function renderActiveWorkspaceRuntime(options: {
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
workspaceSecretStore: options.workspaceSecretStore,
catalogDatabase: options.catalogDatabase,
});
return {
...rendered,
@@ -480,6 +484,7 @@ export async function publishDeterministicRuntimeConfigLease(options: {
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): Promise<DeterministicRuntimeConfigLease> {
const rendered = await renderActiveWorkspaceRuntime(options);
const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
@@ -487,7 +492,11 @@ export async function publishDeterministicRuntimeConfigLease(options: {
const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
const configFingerprintValue = configFingerprint(renderedConfigObject);
const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject);
const inputFingerprintValue = preprocessingInputFingerprint(
rendered.workspaceId,
rendered.workspaceRevision,
renderedConfigObject,
);
const identitySuffix = inputFingerprintValue.slice(7, 23);
const preprocessingRoot = ensureTrustedDirectory(join(
backend/src/workspaces/runtime-renderer.ts
+7 -6
View File
@@ -3,6 +3,7 @@ import { stringify } from "yaml";
import { buildInstallationContract } from "./contracts.js";
import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js";
import type { ResolvedBinding, ResolvedEvidenceBinding, RuntimeBindings } from "./bindings.js";
import { workspaceVectorCollections } from "./vector-collections.js";
export type { RuntimeBindings } from "./bindings.js";
export interface RuntimePaths {
@@ -10,10 +11,8 @@ export interface RuntimePaths {
artifacts: string;
indexes: string;
memory: string;
/** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */
annotations_root?: string;
/** Immutable Catalog projection used as the exclusive runtime relationship source. */
effective_relationships?: string;
/** Current backend-produced projection of the PostgreSQL Metadata Catalog. */
catalog_metadata_snapshot?: string;
}
export interface RuntimeIdentity {
@@ -224,6 +223,7 @@ export function renderRuntimeConfig(
): string {
requireSupportedDescriptor(workspace);
const descriptor = validateWorkspaceDescriptor(workspace);
if (!descriptor.dwh) throw new Error("runtime configuration requires a Catalog database binding");
const contract = buildInstallationContract(descriptor);
const name = (role: "DWH" | "EVIDENCE", suffix: string) => {
const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix);
@@ -243,6 +243,7 @@ export function renderRuntimeConfig(
}
const dwhIdentity = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
const vectorCollections = workspaceVectorCollections(descriptor.workspace.id);
const database = bindings.dwh.transport === "postgres_direct"
? { ...directConnection(bindings.dwh, {
host: name("DWH", "HOST"),
@@ -268,7 +269,7 @@ export function renderRuntimeConfig(
semantic_index: {
vector_store: {
engine: "qdrant",
collection: descriptor.workspace.id,
collections: vectorCollections,
dimensions: semanticRuntime.internalEmbeddingDimensions,
distance: "cosine",
},
@@ -284,7 +285,7 @@ export function renderRuntimeConfig(
vector: {
engine: "qdrant",
base_url: semanticRuntime.internalQdrantUrl,
collection: descriptor.workspace.id,
collections: vectorCollections,
},
embeddings: {
provider: "ollama_internal",
backend/src/workspaces/schema.ts
+34 -7
View File
@@ -53,7 +53,8 @@ interface WorkspaceDwh {
interface WorkspaceBase {
workspace: WorkspaceMetadata;
dwh: WorkspaceDwh;
/** Legacy connection block; current descriptors bind their database through PostgreSQL. */
dwh?: WorkspaceDwh;
diagnostics?: Pick<CanonicalDiagnostics, "dwh_rest">;
}
@@ -328,7 +329,9 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert
}
function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
if (workspace.dwh) {
unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
}
if (workspace.evidence?.source.type === "filesystem") {
const expected = `${workspace.workspace.id}/evidence`;
@@ -341,7 +344,8 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
}
}
if (workspace.diagnostics?.dwh_rest && !workspace.dwh.supported_transports.includes("rest_api")) {
if (workspace.diagnostics?.dwh_rest
&& !workspace.dwh?.supported_transports.includes("rest_api")) {
context.addIssue({
code: "custom",
path: ["diagnostics", "dwh_rest"],
@@ -351,7 +355,7 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
}
const WorkspaceV4Schema = z.object({
dwh: dwhSchema,
dwh: dwhSchema.optional(),
evidence: workspaceEvidenceSchema.optional(),
diagnostics: z.object({
dwh_rest: dwhRestDiagnostic.optional(),
@@ -363,7 +367,7 @@ const WorkspaceV4Schema = z.object({
}).strict().superRefine(workspaceInvariants);
const WorkspaceDescriptorSchema = WorkspaceV4Schema;
export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
function parseWorkspaceDocument(source: string): unknown {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
const document = documents[0];
@@ -371,10 +375,30 @@ export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
throw new Error(`Invalid workspace YAML: ${[...document.errors, ...document.warnings]
.map((error) => error.message).join("; ")}`);
}
return validateWorkspaceDescriptor(document.toJSON());
return document.toJSON();
}
/** Deterministically removes the two installation-owned v3 blocks without altering workspace data. */
export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
const value = parseWorkspaceDocument(source);
assertAuthoredWorkspaceIsDatabaseFree(value);
return validateWorkspaceDescriptor(value);
}
/** Parses an ephemeral, generated core runtime descriptor that may contain a Catalog binding. */
export function parseRuntimeWorkspaceYaml(source: string): WorkspaceDescriptor {
return validateWorkspaceDescriptor(parseWorkspaceDocument(source));
}
function assertAuthoredWorkspaceIsDatabaseFree(workspace: unknown): void {
if (workspace !== null && typeof workspace === "object"
&& ("dwh" in workspace || "diagnostics" in workspace)) {
throw new Error(
"Workspace YAML must not contain database configuration; use the PostgreSQL Metadata Catalog",
);
}
}
/** Build a database-free v4 descriptor; legacy database/configuration fields are not carried over. */
export function migrateWorkspaceV3Yaml(source: string): string {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
@@ -388,6 +412,8 @@ export function migrateWorkspaceV3Yaml(source: string): string {
throw new Error("Workspace migration requires schema version 3");
}
metadata.schema_version = 4;
delete value.dwh;
delete value.diagnostics;
delete value.semantic_index;
delete value.llm_policy;
return serializeWorkspaceYaml(validateWorkspaceDescriptor(value));
@@ -423,6 +449,7 @@ export function resolveDiagnosticUrl(baseUrl: string, path: string): URL {
export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string {
const canonical = validateOperationalWorkspace(workspace);
assertAuthoredWorkspaceIsDatabaseFree(canonical);
return stringify(canonical, { lineWidth: 0, sortMapEntries: true });
}
backend/src/workspaces/secret-requirements.ts
+6 -2
View File
@@ -101,7 +101,8 @@ function selectedTransport(
descriptor: WorkspaceDescriptor,
variables: readonly InstallationVariable[],
env: NodeJS.ProcessEnv,
): DwhTransport {
): DwhTransport | undefined {
if (!descriptor.dwh) return undefined;
const transportVariable = variables.find(({ role, suffix }) => role === "DWH" && suffix === "TRANSPORT");
const value = transportVariable === undefined ? undefined : env[transportVariable.name];
return isTransport(value) && descriptor.dwh.supported_transports.includes(value)
@@ -136,11 +137,14 @@ export function discoverWorkspaceSecretRequirements(
const variables = buildInstallationContract(descriptor).variables;
const transport = selectedTransport(descriptor, variables, env);
const restHasNoAuthentication = transport === "rest_api" && descriptor.diagnostics?.dwh_rest?.auth === "none";
const requiredDwh = new Set(restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport]);
const requiredDwh = new Set(
transport === undefined || restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport],
);
const requirements: WorkspaceSecretRequirement[] = [];
for (const variable of variables) {
if (variable.role === "DWH") {
if (transport === undefined) continue;
if (variable.transports !== undefined && !variable.transports.includes(transport)) continue;
const requirement = requirementFor(variable, requiredDwh.has(variable.suffix));
if (requirement !== undefined && requirement.required) requirements.push(requirement);
backend/src/workspaces/vector-collections.ts
+20
View File
@@ -0,0 +1,20 @@
export interface WorkspaceVectorCollections {
reference: string;
memory: string;
}
/**
* Physical Qdrant namespaces owned by one workspace.
*
* Reference data is replaceable preprocessing output. Memory is durable runtime
* state and deliberately has a separate lifecycle.
*/
export function workspaceVectorCollections(workspaceId: string): WorkspaceVectorCollections {
if (!/^[a-z][a-z0-9-]{2,62}$/.test(workspaceId)) {
throw new Error("workspace id is invalid");
}
return {
reference: `${workspaceId}-reference`,
memory: `${workspaceId}-memory`,
};
}
backend/test/catalog-databases-routes.test.ts
+6 -14
View File
@@ -97,31 +97,23 @@ const fleetSnapshot: ObservedSchemaSnapshot = {
}],
};
test("lists every YAML workspace and creates its one database configuration", async () => {
test("lists every workspace and creates its Catalog database configuration", async () => {
const { app, secretStore } = setup();
const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(initial.statusCode).toBe(200);
expect(initial.json()).toMatchObject([{
workspaceId: "psd-clinical",
configured: false,
databaseName: "warehouse",
databaseName: "",
workspaceRevision: { commit: revision.commit, blob: revision.blob },
workspaceEvidence: { sourceType: null, state: "not_declared" },
runtimeBinding: {
transport: "postgres_direct",
configurationState: "configuration_required",
sessionTransportSupported: true,
},
runtimeBinding: null,
}]);
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" });
const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(runtimeReady.json()).toMatchObject([{
runtimeBinding: { configurationState: "ready", sessionTransportSupported: true },
runtimeBinding: null,
}]);
const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
@@ -166,7 +158,7 @@ test("projects remote Evidence credential state without conflating catalog secre
}]);
});
test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
test("lists orphaned records and keeps the REST diagnostic path in the Catalog", async () => {
const { app, repository } = setup();
await repository.create({
workspaceId: "removed-workspace",
@@ -186,7 +178,7 @@ test("lists orphaned records and takes the REST diagnostic path from workspace Y
},
});
expect(created.statusCode).toBe(201);
expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
expect(created.json()).toMatchObject({ binding: { restPath: "/client-controlled" } });
const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
expect(rows).toEqual(expect.arrayContaining([
backend/test/catalog-description-generation-routes.test.ts
+12 -7
View File
@@ -577,7 +577,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
expect.objectContaining({
sequence: 3,
level: "info",
message: `Generated description for Catalog Column ${column.id}.`,
message: 'Generated description for Column "patients.birth_date".',
}),
expect.objectContaining({ sequence: 4, level: "info", message: "Description generation completed." }),
]);
@@ -934,7 +934,10 @@ test("generates selected Catalog Columns in caller order through sequential batc
const events = await repository.listDescriptionGenerationEvents(run.id);
expect(events.map((event) => event.sequence)).toEqual(Array.from({ length: 14 }, (_, index) => index + 1));
expect(events.slice(2, -1).map((event) => event.message)).toEqual(
orderedIds.map((targetId) => `Generated description for Catalog Column ${targetId}.`),
orderedIds.map((targetId) => {
const target = columns.find((column) => column.id === targetId)!;
return `Generated description for Column "patients.${target.name}".`;
}),
);
} finally {
pending[0]!.resolve(responseFor(orderedIds.slice(0, 10), 0));
@@ -1803,7 +1806,9 @@ test("retains completed batch writes when a later batch response is malformed",
});
const events = await repository.listDescriptionGenerationEvents(run.id);
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
originalColumns.slice(0, 10).map(
(target) => `Generated description for Column "patients.${target.name}".`,
),
);
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
expect.objectContaining({
@@ -1812,7 +1817,7 @@ test("retains completed batch writes when a later batch response is malformed",
);
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
message: `The model response did not match the required schema. Affected target: Column "patients.${originalColumns[10]!.name}".`,
}));
} finally {
await app.close();
@@ -2301,7 +2306,7 @@ test("generates selected Catalog Tables with structural column context and local
expect((await repository.listDescriptionGenerationEvents(run.id)).map((event) => event.message)).toEqual([
"Description generation queued.",
"Description generation started.",
`Stored non-generatable result for Catalog Table ${table.id}.`,
'Stored non-generatable result for Table "patients".',
"Description generation completed.",
]);
} finally {
@@ -2445,7 +2450,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
expect(`${JSON.stringify(run)}${events.body}`).not.toContain(sensitiveDiagnostic);
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model provider request failed. Affected Catalog Column target: ${column.id}.`,
message: 'The model provider request failed. Affected target: Column "patients.birth_date".',
}));
} finally {
await app.close();
@@ -2536,7 +2541,7 @@ test.each([
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
expect.objectContaining({
level: "error",
message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
message: `${failureMessage} Affected targets: Column "patients.first_column", Column "patients.second_column".`,
}),
);
} finally {
backend/test/catalog-description-generation.integration.test.ts
+5 -1
View File
@@ -13,10 +13,12 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
import { loadConfig } from "../src/config.js";
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
@@ -52,10 +54,12 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
await upSensitiveDataFlag(db);
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
await upLogicalRelationships(db);
await upAiTokenUsage(db);
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
await upCatalogPreprocessingState(db);
const repository = new KyselyCatalogRepository(db);
const database = await repository.create({
workspaceId: "psd-clinical",
@@ -282,7 +286,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
expect(events.statusCode).toBe(200);
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
message: 'The model provider request failed. Affected target: Column "patients.status".',
}));
expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
backend/test/catalog-repository.integration.test.ts
+91 -1
View File
@@ -18,6 +18,7 @@ import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usa
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
@@ -58,6 +59,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
await upCatalogPreprocessingState(db);
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
.select(["engine", "modelId", "policyVersion", "unknown"])
.where("id", "=", historicalSuggestionRunId)
@@ -232,6 +234,77 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
expect(await repository.listSyncRuns(created.id)).toEqual([
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
]);
const lockedDatabase = await repository.create({
workspaceId: "preprocessing-lock",
engine: "postgres",
databaseName: "warehouse",
schema: "public",
binding: {
transport: "postgres_direct", host: "lock.internal", port: 5432, username: "reader",
},
});
await repository.applySchemaSync(
lockedDatabase.id,
lockedDatabase.version,
"all",
[],
{
schemaVersion: 1,
capabilities: { tables: "available", columns: "available", relationships: "available" },
tables: [], columns: [], relationships: [],
},
);
const beforePreprocessing = (await repository.get(lockedDatabase.id))!;
const preprocessing = await repository.beginPreprocessing(
"preprocessing-lock",
"sha256:" + "1".repeat(64),
);
expect(preprocessing).toMatchObject({ kind: "started" });
await expect(db.insertInto("catalogTables").values({
id: randomUUID(),
databaseId: lockedDatabase.id,
name: "blocked_write",
sourceComment: null,
description: null,
generatedDescription: null,
}).execute()).rejects.toThrow("catalog preprocessing is running");
await expect(repository.createDescriptionGenerationRun(
lockedDatabase.id,
"all",
"openai/gpt-5-mini",
"en",
0,
)).rejects.toThrow("catalog preprocessing is running");
await repository.recordTest(lockedDatabase.id, lockedDatabase.version, {
connectionStatus: "reachable",
testedVersion: lockedDatabase.version,
lastTestedAt: "2026-01-01T00:00:00Z",
});
expect((await repository.get(lockedDatabase.id))?.metadataContentRevision)
.toBe(beforePreprocessing.metadataContentRevision);
await expect(repository.finishPreprocessing(
"preprocessing-lock",
beforePreprocessing.metadataContentRevision,
"sha256:" + "1".repeat(64),
{ status: "succeeded" },
)).resolves.toMatchObject({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: beforePreprocessing.metadataContentRevision,
});
await db.insertInto("catalogTables").values({
id: randomUUID(),
databaseId: lockedDatabase.id,
name: "allowed_after_preprocessing",
sourceComment: null,
description: null,
generatedDescription: null,
}).execute();
await expect(repository.get(lockedDatabase.id)).resolves.toMatchObject({
preprocessingStatus: "failed",
metadataContentRevision: beforePreprocessing.metadataContentRevision + 1,
});
expect(await repository.delete(lockedDatabase.id, lockedDatabase.version)).toBe(true);
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
expect(await repository.delete(created.id, 2)).toBe(true);
expect(await repository.list()).toEqual([]);
@@ -355,7 +428,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
}
}, 60_000);
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected and database-wide descriptions", async () => {
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
const db = new Kysely<CatalogDatabase>({
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
@@ -449,6 +522,22 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel
description: "Still curated visits",
generatedDescription: "Generated visits",
});
expect(await repository.consolidateGeneratedDescriptions(
database.id, "database", [],
)).toEqual({ copied: 3, skipped: 1 });
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
description: "Generated visits",
generatedDescription: "Generated visits",
});
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
description: "Generated id",
generatedDescription: "Generated id",
});
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
description: "Keep patient reference",
generatedDescription: null,
});
} finally {
await db.destroy();
await container.stop();
@@ -473,6 +562,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
await upCatalogPreprocessingState(db);
const repository = new KyselyCatalogRepository(db);
const firstDatabase = await repository.create({
workspaceId: "generation-one",
backend/test/catalog-runtime-binding.test.ts
+105
View File
@@ -0,0 +1,105 @@
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test } from "vitest";
import { resolveCatalogRuntimeBinding } from "../src/catalog/runtime-binding.js";
import type { WorkspaceDatabase } from "../src/catalog/types.js";
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
const roots: string[] = [];
afterEach(() => {
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});
test("projects a Catalog database into a runtime without database data in workspace YAML", () => {
const root = mkdtempSync(join(tmpdir(), "thoth-catalog-runtime-"));
roots.push(root);
const secretStore = new WorkspaceSecretStore({
root: join(root, "vault"),
runtimeRoot: join(root, "runtime"),
installationId: "test",
});
secretStore.putMany("sales", {
"catalog.dwh.password": "catalog-password",
"evidence.signed_urls": '["https://signed.example.test/evidence"]',
});
const workspace: WorkspaceDescriptor = {
workspace: {
schema_version: 4,
id: "sales",
name: "Sales",
language: "en",
},
evidence: {
schema_version: 1,
source: {
type: "http",
uris: ["https://evidence.example.test/guide.md"],
authentication: "signed_urls_file",
connect_timeout_ms: 1_000,
read_timeout_ms: 5_000,
max_bytes: 10_000,
max_redirects: 2,
allow_private_hosts: false,
max_cache_bytes: 20_000,
},
policy: { max_chunk_chars: 4_000, retain_published_generations: 1 },
},
};
const database: WorkspaceDatabase = {
id: "database-1",
workspaceId: "sales",
engine: "postgres",
databaseName: "warehouse",
schema: "analytics",
version: 3,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
binding: {
transport: "postgres_direct",
host: "db.internal",
port: 5432,
username: "reader",
},
connectionStatus: "reachable",
metadataContentRevision: 7,
preprocessingStatus: "failed",
};
const lease = resolveCatalogRuntimeBinding({
workspace,
database,
environment: {},
secretRoots: [],
secretStore,
});
const passwordPath = lease.bindings.dwh.values.THT_WS_SALES_DWH_PASSWORD_FILE;
const evidencePath = lease.bindings.evidence.values.THT_WS_SALES_EVIDENCE_SIGNED_URLS_FILE;
try {
expect(workspace.dwh).toBeUndefined();
expect(lease.workspace.dwh).toMatchObject({
database: "warehouse",
schema: "analytics",
supported_transports: ["postgres_direct"],
});
expect(lease.bindings.dwh).toMatchObject({
transport: "postgres_direct",
missing: [],
values: {
THT_WS_SALES_DWH_HOST: "db.internal",
THT_WS_SALES_DWH_PORT: "5432",
THT_WS_SALES_DWH_USER: "reader",
},
});
expect(readFileSync(passwordPath, "utf8")).toBe("catalog-password");
expect(readFileSync(evidencePath, "utf8")).toContain("signed.example.test");
} finally {
lease.release();
}
expect(existsSync(passwordPath)).toBe(false);
expect(existsSync(evidencePath)).toBe(false);
});
backend/test/catalog-schema-routes.test.ts
+21 -4
View File
@@ -394,12 +394,19 @@ test("consolidates non-empty generated column descriptions and preserves curated
expect(scan).not.toHaveBeenCalled();
});
test("consolidates generated descriptions for every column in a database", async () => {
test("consolidates generated descriptions for every table and column in a database", async () => {
const { app, repository, database, scan } = await setup();
await seedCatalog(repository, database);
const tables = await repository.listTables(database.id);
const patients = tables.find((table) => table.name === "patients")!;
const visits = tables.find((table) => table.name === "visits")!;
await repository.updateTableMetadata(
database.id,
patients.id,
patients.version,
"Curated patients",
"Generated patients",
);
const patientId = (await repository.listColumns(database.id, patients.id))[0]!;
const visitColumns = await repository.listColumns(database.id, visits.id);
const visitId = visitColumns.find((column) => column.name === "id")!;
@@ -432,11 +439,16 @@ test("consolidates generated descriptions for every column in a database", async
const response = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
payload: { target: "database_columns" },
payload: { target: "database" },
});
expect(response.statusCode).toBe(200);
expect(response.json()).toEqual({ copied: 2, skipped: 1 });
expect(response.json()).toEqual({ copied: 3, skipped: 2 });
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
description: "Generated patients",
generatedDescription: "Generated patients",
version: patients.version + 2,
});
expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({
description: "Generated patient identifier",
generatedDescription: "Generated patient identifier",
@@ -529,6 +541,11 @@ test("strictly validates description consolidation database and target ids", asy
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
payload: { target: "tables", targetIds: [table.id], unexpected: true },
}),
app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
payload: { target: "database", targetIds: [table.id] },
}),
app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
@@ -536,7 +553,7 @@ test("strictly validates description consolidation database and target ids", asy
}),
]);
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400]);
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400, 400]);
for (const response of responses) {
expect(response.json()).toEqual({
code: "description_consolidation_invalid",
backend/test/effective-config.test.ts
+17 -5
View File
@@ -41,7 +41,10 @@ function directRendered(): Record<string, unknown> {
vector: {
engine: "qdrant",
base_url: "http://qdrant:6333",
collection: "psd-clinical",
collections: {
reference: "psd-clinical-reference",
memory: "psd-clinical-memory",
},
dimensions: 1024,
distance: "cosine",
collection_lifecycle: "require_existing",
@@ -103,10 +106,10 @@ function restRendered(): Record<string, unknown> {
}
const directCanonical =
`{"schemaVersion":2,"dwh":{` +
`{"schemaVersion":3,"dwh":{` +
`"engine":"postgres","database":"postgres","schema":"datawarehouse",` +
`"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` +
`"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` +
`"vector":{"collections":{"reference":"psd-clinical-reference","memory":"psd-clinical-memory"},"dimensions":1024,"distance":"cosine"},` +
`"embedding":{"id":"ollama/qwen3-embedding:0.6b","model":"qwen3-embedding:0.6b","dimensions":1024},` +
`"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` +
`"indexes":"/data/sessions/psd-clinical/indexes"}}`;
@@ -128,7 +131,7 @@ test("canonical effective config excludes secrets, evidence, session storage, an
expect(json).not.toContain("sources");
expect(json).not.toContain("collection_lifecycle");
expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity
expect(json).not.toContain("memory");
expect(json).not.toContain('"memory":"/data');
expect(json).not.toContain('"sessions"');
});
@@ -190,7 +193,16 @@ test("DWH-affecting changes alter the effective config identity", () => {
const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } };
expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore);
const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record<string, any>).vector, collection: "other" } } };
const changedCollection = {
...base,
resources: {
...base.resources,
vector: {
...(base.resources as Record<string, any>).vector,
collections: { reference: "other-reference", memory: "other-memory" },
},
},
};
expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore);
const changedEmbeddingIdentity = { ...base, resources: { ...base.resources, embeddings: { ...(base.resources as Record<string, any>).embeddings, model: "other-embedding" } } };
backend/test/registry-annotations.test.ts
-5
View File
@@ -19,11 +19,6 @@ const validYaml = `workspace:
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
`;
async function git(cwd: string, args: string[]): Promise<string> {
backend/test/registry-evidence.test.ts
-5
View File
@@ -24,11 +24,6 @@ const descriptor = `workspace:
id: research
name: Research
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: filesystem
backend/test/routes-sessions.test.ts
+90 -13
View File
@@ -12,6 +12,7 @@ import { PiProcessManager } from "../src/pi/pi-process-manager.js";
import { validateDeclarativePiConfig } from "../src/pi/managed-config.js";
import Fastify from "fastify";
import { sessionRoutes } from "../src/routes/sessions.js";
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
const SCRIPT = path.resolve("../harness/tests/fake_pi/scripts/f1_disambiguation.json");
@@ -517,7 +518,7 @@ test("creates a session from the active immutable workspace revision", async ()
}));
});
test("hands one effective relationship snapshot to both retrieval and Pi", async () => {
test("hands one Catalog-backed runtime to both retrieval and Pi", async () => {
const effective = JSON.stringify({
schemaVersion: 1,
workspaceId: "default",
@@ -556,10 +557,9 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
});
expect(response.statusCode).toBe(200);
expect(render).toHaveBeenCalledWith("default");
expect(render).not.toHaveBeenCalled();
expect(acquireWorkspaceRuntime).toHaveBeenCalledWith(
expect.stringContaining("/default.yaml"),
effective,
);
expect(createFor).toHaveBeenCalledWith(
"effective-map",
@@ -574,27 +574,103 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
);
});
test("rejects an SSH-only workspace before persisting or starting a session", async () => {
test("refuses core admission when the workspace preprocessing fingerprint is stale", async () => {
const sessionNew = vi.fn();
const workspaceInputFingerprint = vi.fn(async () => "sha256:current");
const revision = {
id: "default",
commit: "a".repeat(40),
blob: "b".repeat(40),
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/default.yaml`,
};
const catalogRepository = new MemoryCatalogRepository();
const database = await catalogRepository.create({
workspaceId: "default",
engine: "postgres",
databaseName: "warehouse",
schema: "public",
binding: {
transport: "postgres_direct",
host: "db.internal",
port: 5432,
username: "reader",
},
});
await catalogRepository.applySchemaSync(database.id, database.version, "all", [], {
schemaVersion: 1,
capabilities: { tables: "available", columns: "available", relationships: "available" },
tables: [],
columns: [],
relationships: [],
});
const started = await catalogRepository.beginPreprocessing("default", "sha256:previous");
expect(started.kind).toBe("started");
await catalogRepository.finishPreprocessing(
"default",
0,
"sha256:previous",
{ status: "succeeded" },
);
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: { sessionNew, workspaceInputFingerprint } as any,
catalogRepository,
workspaceRegistry: {
acquireSessionRevision: async () => ({
workspace: operationalWorkspace("default"),
revision,
abort: async () => {},
markPersisted: async () => {},
}),
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "default", thinking: "low" }) as any,
});
const response = await app.inject({
method: "POST",
url: "/sessions",
payload: { question: "q", workspaceId: "default" },
});
expect(response.statusCode).toBe(409);
expect(response.json()).toMatchObject({ code: "preprocessing_required" });
expect(workspaceInputFingerprint).toHaveBeenCalledWith(revision.snapshotPath);
expect(sessionNew).not.toHaveBeenCalled();
});
test("rejects an SSH-only Catalog binding before persisting or starting a session", async () => {
const sessionNew = vi.fn(async () => ({ id: "must-not-exist" }));
const workspaceInputFingerprint = vi.fn(async () => "sha256:unused");
const ensure = vi.fn(async () => ({ ok: true }));
const createFor = vi.fn();
const abort = vi.fn(async () => {});
const markPersisted = vi.fn(async () => {});
const catalogRepository = new MemoryCatalogRepository();
await catalogRepository.create({
workspaceId: "ssh-workspace",
engine: "postgres",
databaseName: "postgres",
schema: "public",
binding: {
transport: "ssh_tunnel",
username: "reader",
sshHost: "bastion.internal",
sshPort: 22,
sshUsername: "tunnel",
sshTargetHost: "postgres.internal",
sshTargetPort: 5432,
},
});
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: { sessionNew, searchPack: async () => {} } as any,
thtRunner: { sessionNew, searchPack: async () => {}, workspaceInputFingerprint } as any,
catalogRepository,
readiness: { ensure } as any,
mgr: { get: () => undefined, createFor } as any,
getSettings: () => ({ workspace: "ssh-workspace" }) as any,
workspaceRuntimeSupport: vi.fn(() => false),
workspaceRuntimeSupport: vi.fn(() => true),
workspaceRegistry: {
acquireSessionRevision: vi.fn(async () => ({
workspace: {
workspace: { schema_version: 4, id: "ssh-workspace", name: "SSH", language: "en" },
dwh: {
engine: "postgres", database: "postgres", schema: "public",
supported_transports: ["ssh_tunnel"],
},
},
workspace: operationalWorkspace("ssh-workspace"),
revision: {
id: "ssh-workspace", commit: "a".repeat(40), blob: "b".repeat(40),
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/ssh-workspace.yaml`,
@@ -610,6 +686,7 @@ test("rejects an SSH-only workspace before persisting or starting a session", as
expect(response.statusCode).toBe(409);
expect(response.json()).toMatchObject({ code: "workspace_not_activatable" });
expect(ensure).not.toHaveBeenCalled();
expect(workspaceInputFingerprint).not.toHaveBeenCalled();
expect(sessionNew).not.toHaveBeenCalled();
expect(createFor).not.toHaveBeenCalled();
expect(abort).toHaveBeenCalledOnce();
backend/test/routes-workspaces.test.ts
+2 -1
View File
@@ -500,8 +500,9 @@ async function createRealRouteFixture() {
await git(author, ["init", "--initial-branch=main"]);
await git(author, ["config", "user.name", "Workspace Route Test"]);
await git(author, ["config", "user.email", "workspace-route@example.invalid"]);
const { dwh: _runtimeDwh, diagnostics: _runtimeDiagnostics, ...authoredWorkspace } = workspace;
const descriptor: CanonicalWorkspace = {
...workspace,
...authoredWorkspace,
evidence: {
source: {
type: "filesystem",
backend/test/tht-qdrant-readiness.test.ts
+5 -2
View File
@@ -52,8 +52,11 @@ test("Qdrant readiness uses only the internal URL and accepts the exact collecti
const request = vi.fn(async () => response(200, collection()));
await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ ok: true, state: "ready" });
expect(request).toHaveBeenCalledOnce();
expect(request.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd");
expect(request).toHaveBeenCalledTimes(2);
expect(request.mock.calls.map((call) => call[0])).toEqual([
"http://qdrant:6333/collections/psd-reference",
"http://qdrant:6333/collections/psd-memory",
]);
expect(request.mock.calls[0][1]).toMatchObject({ method: "GET", signal: expect.any(AbortSignal) });
});
backend/test/workspace-maintenance.test.ts
+18 -30
View File
@@ -30,11 +30,11 @@ function ok(operation: string): WorkspaceOperationResult {
};
}
test("entrypoint emits exactly one pristine JSON document and maps success/block/failure exits", async () => {
test("entrypoint emits exactly one pristine JSON document and maps success/failure exits", async () => {
const service = {
inspect: vi.fn(async () => ok("inspect")),
preprocessDwh: vi.fn(async () => ({ ...ok("preprocess dwh"), status: "blocked", code: "manual_review_required" as const })),
run: vi.fn(async () => ({ ...ok("preprocess run"), status: "failed", code: "semantic_index_incompatible" as const })),
clear: vi.fn(async () => ok("preprocess clear")),
} as any;
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
@@ -42,13 +42,13 @@ test("entrypoint emits exactly one pristine JSON document and maps success/block
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "inspect", code: "ok" });
expect(inspectIo.stderr.join("")).toBe("");
const blockedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-dwh"], service, blockedIo)).toBe(3);
expect(JSON.parse(blockedIo.stdout.join(""))).toMatchObject({ code: "manual_review_required" });
const failedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-run"], service, failedIo)).toBe(1);
expect(JSON.parse(failedIo.stdout.join(""))).toMatchObject({ code: "semantic_index_incompatible" });
const clearIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-clear"], service, clearIo)).toBe(0);
expect(JSON.parse(clearIo.stdout.join(""))).toMatchObject({ operation: "preprocess clear", code: "ok" });
});
test("malformed stdin, unknown commands, and extra fields fail with exit 2 but still return bounded JSON", async () => {
@@ -84,31 +84,19 @@ test("raw exception text is redacted from stderr and stdout remains within the p
expect(captured.stderr.join("")).not.toContain("SELECT *");
});
test("vector-inspect and vector-rebuild dispatch to the service with the exact envelope", async () => {
test("retired partial preprocessing commands are rejected without dispatch", async () => {
const service = {
vectorInspect: vi.fn(async () => ok("vector inspect")),
vectorRebuild: vi.fn(async () => ok("vector rebuild")),
preprocessDwh: vi.fn(),
vectorInspect: vi.fn(),
vectorRebuild: vi.fn(),
} as any;
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-inspect"], service, inspectIo)).toBe(0);
expect(service.vectorInspect).toHaveBeenCalledWith({ workspaceId: "psd-clinical" });
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "vector inspect", code: "ok" });
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(0);
expect(service.vectorRebuild).toHaveBeenCalledWith({
workspaceId: "psd-clinical",
collection: "psd-clinical",
confirm: "psd-clinical",
destroy: true,
});
});
test("vector-rebuild without exact confirmation is refused by the service", async () => {
const service = {
vectorRebuild: vi.fn(async () => ({ ...ok("vector rebuild"), status: "failed", code: "semantic_index_incompatible" as const })),
} as any;
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "other", confirm: "other", destroy: true }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(1);
for (const command of ["preprocess-dwh", "vector-inspect", "vector-rebuild"]) {
const captured = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", command], service, captured)).toBe(2);
expect(JSON.parse(captured.stdout.join(""))).toMatchObject({ status: "failed", operation: command });
}
expect(service.preprocessDwh).not.toHaveBeenCalled();
expect(service.vectorInspect).not.toHaveBeenCalled();
expect(service.vectorRebuild).not.toHaveBeenCalled();
});
backend/test/workspace-preprocessing-routes.test.ts
+259
View File
@@ -0,0 +1,259 @@
import Fastify from "fastify";
import { expect, test, vi } from "vitest";
import type { PrincipalContext } from "../src/auth/principal.js";
import type { CatalogRepository, WorkspaceDatabase } from "../src/catalog/types.js";
import {
readWorkspacePreprocessingStatus,
workspacePreprocessingRoutes,
type WorkspacePreprocessingRouteDeps,
} from "../src/routes/workspace-preprocessing.js";
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
const admin: PrincipalContext = {
issuer: "test",
subject: "admin",
roles: ["admin"],
permissions: ["session.use", "database.manage"],
isAdmin: true,
};
function database(overrides: Partial<WorkspaceDatabase> = {}): WorkspaceDatabase {
return {
id: "49b6491a-78bb-4cee-b27b-0efaf4419774",
workspaceId: "catalog-workspace",
engine: "postgres",
databaseName: "warehouse",
schema: "analytics",
version: 4,
createdAt: "2026-09-05T10:00:00.000Z",
updatedAt: "2026-09-05T10:00:00.000Z",
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
connectionStatus: "reachable",
schemaSyncedVersion: 4,
metadataContentRevision: 18,
preprocessingStatus: "failed",
...overrides,
};
}
function routeDeps(
current: () => WorkspaceDatabase | undefined,
overrides: Partial<WorkspacePreprocessingRouteDeps> = {},
): WorkspacePreprocessingRouteDeps {
const repository = {
getByWorkspace: vi.fn(async () => current()),
listSyncRuns: vi.fn(async () => []),
getActiveDescriptionGenerationRun: vi.fn(async () => undefined),
listSensitivityAnalysisRuns: vi.fn(async () => []),
} as unknown as CatalogRepository;
const registry = {
read: vi.fn(async () => ({
workspace: {
workspace: {
schema_version: 4,
id: "catalog-workspace",
name: "Catalog workspace",
language: "en",
},
},
revision: {
id: "catalog-workspace",
commit: "a".repeat(40),
blob: "b".repeat(40),
snapshotPath: "/data/workspaces/catalog-workspace.yaml",
},
})),
} as unknown as WorkspaceRegistry;
return {
repository,
registry,
service: {
run: vi.fn(async () => ({ status: "succeeded" })),
clear: vi.fn(async () => ({ status: "succeeded" })),
} as never,
inputFingerprint: {
workspaceInputFingerprint: vi.fn(async () => "sha256:current"),
} as never,
...overrides,
};
}
test("reports ready only when the Catalog revision and runtime fingerprint are current", async () => {
const ready = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
});
await expect(readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => ready),
)).resolves.toMatchObject({
state: "ready",
actionable: true,
clearable: true,
detail: "Catalog revision 18 is indexed.",
});
ready.preprocessingInputFingerprint = "sha256:old";
await expect(readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => ready),
)).resolves.toMatchObject({
state: "required",
actionable: true,
clearable: true,
detail: "Catalog revision 18 is not indexed.",
});
});
test("explains a current blocked prerequisite without inventing a run log", async () => {
const status = await readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => database({ schemaSyncedVersion: 3 })),
);
expect(status).toMatchObject({
state: "blocked",
actionable: false,
clearable: true,
detail: "Catalog synchronization is required.",
reason: "Database configuration v4 is newer than the latest Catalog synchronization v3.",
nextStep: "Open Database management and run Synchronize schema.",
});
expect(status).not.toHaveProperty("lastFailure");
});
test("exposes only the latest sanitized failed-run diagnostic", async () => {
const status = await readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => database({
preprocessingStatus: "failed",
preprocessingErrorCode: "schema_index_failed",
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
})),
);
expect(status).toMatchObject({
state: "failed",
actionable: true,
clearable: true,
reason: expect.stringContaining("schema indexing worker"),
lastFailure: {
stage: "schema_index",
errorCode: "schema_index_failed",
finishedAt: "2026-09-05T10:04:00.000Z",
},
});
expect(status).not.toHaveProperty("history");
});
test("reports the durable phase of the active preprocessing run", async () => {
const status = await readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(
() => database({ preprocessingStatus: "running" }),
{
readLatestJob: () => ({
status: "active",
completedStages: ["catalog_snapshot"],
}) as never,
},
),
);
expect(status).toMatchObject({
state: "running",
detail: "Building schema vectors and LSH indexes.",
progress: { stage: "schema_index", step: 2, totalSteps: 4 },
});
});
test("explicitly reruns preprocessing when the current Catalog input is already ready", async () => {
const ready = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
});
const deps = routeDeps(() => ready);
const app = Fastify();
app.addHook("preHandler", async (request) => { request.principal = admin; });
workspacePreprocessingRoutes(app, deps);
const response = await app.inject({
method: "POST",
url: "/workspaces/catalog-workspace/preprocessing",
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
expect(deps.service.run).toHaveBeenCalledTimes(1);
await app.close();
});
test("runs preprocessing once from the sanctioned admin endpoint and returns the new state", async () => {
let current = database();
const deps = routeDeps(() => current, {
service: {
run: vi.fn(async () => {
current = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
});
return { status: "succeeded" } as never;
}),
clear: vi.fn(async () => ({ status: "succeeded" } as never)),
},
});
const app = Fastify();
app.addHook("preHandler", async (request) => { request.principal = admin; });
workspacePreprocessingRoutes(app, deps);
const response = await app.inject({
method: "POST",
url: "/workspaces/catalog-workspace/preprocessing",
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
expect(deps.service.run).toHaveBeenCalledTimes(1);
await app.close();
});
test("clears only derived preprocessing data from the sanctioned admin endpoint", async () => {
let current = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
});
const deps = routeDeps(() => current, {
service: {
run: vi.fn(),
clear: vi.fn(async () => {
current = database({
preprocessingStatus: "failed",
preprocessingErrorCode: "derived_data_cleared",
preprocessingFinishedAt: "2026-09-05T10:05:00.000Z",
});
return { status: "succeeded" } as never;
}),
},
});
const app = Fastify();
app.addHook("preHandler", async (request) => { request.principal = admin; });
workspacePreprocessingRoutes(app, deps);
const response = await app.inject({
method: "DELETE",
url: "/workspaces/catalog-workspace/preprocessing",
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
state: "required",
clearable: false,
detail: "Reference vectors and LSH are empty. Memory is preserved.",
});
expect(deps.service.clear).toHaveBeenCalledTimes(1);
await app.close();
});
backend/test/workspace-preprocessing-service.test.ts
+204 -420
View File
@@ -2,9 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test, vi } from "vitest";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { PreprocessingStateStore } from "../src/workspaces/preprocessing-state.js";
import { syncAnnotations } from "../src/workspaces/annotations-sync.js";
import { validateWorkspaceDescriptor } from "../src/workspaces/schema.js";
import {
WorkspacePreprocessingService,
type ChildProcessRequest,
@@ -16,73 +14,25 @@ afterEach(() => {
roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
});
const semanticRuntime = {
internalQdrantUrl: "http://qdrant:6333",
internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
internalEmbeddingModel: "qwen3-embedding:0.6b",
internalEmbeddingDimensions: 1024,
};
const workspace = validateWorkspaceDescriptor({
workspace: {
schema_version: 4,
id: "catalog-workspace",
name: "Catalog only",
language: "en",
},
});
const baseWorkspace = parseWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Runtime Lease
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
`);
const filesystemWorkspace = parseWorkspaceYaml(`${baseWorkspace ? '' : ''}workspace:
schema_version: 4
id: fs-workspace
name: Filesystem
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: filesystem
uri: fs-workspace/evidence
`);
const privateHttpWorkspace = parseWorkspaceYaml(`workspace:
schema_version: 4
id: http-workspace
name: Http
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: http
uris: [http://127.0.0.1/private.md]
authentication: none
connect_timeout_ms: 1000
read_timeout_ms: 2000
max_bytes: 100
max_redirects: 0
allow_private_hosts: true
max_cache_bytes: 100
`);
function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id) {
function runtime() {
return {
workspace,
workspaceId,
workspaceId: "catalog-workspace",
workspaceRevision: "a".repeat(40),
descriptorBlob: "b".repeat(40),
catalogBlob: "c".repeat(40),
configLease: {
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}-identitysuffix.yaml`,
workspaceId,
path: `/data/sessions/catalog-workspace/preprocessing/runtime-config/${"a".repeat(40)}.yaml`,
workspaceId: "catalog-workspace",
workspaceRevision: "a".repeat(40),
descriptorBlob: "b".repeat(40),
catalogBlob: "c".repeat(40),
@@ -90,21 +40,32 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
bindingDigest: "sha256:bindings",
semanticQdrantUrl: "http://qdrant:6333",
effectiveConfig: {
schemaVersion: 2,
schemaVersion: 3,
dwh: {
engine: "postgres",
database: "analytics",
schema: "mart",
database: "warehouse",
schema: "analytics",
transport: "postgres_direct",
host: "dwh.internal",
host: "db.internal",
port: 5432,
user: "reader",
},
vector: { collection: workspaceId, dimensions: 1024, distance: "cosine" },
embedding: { id: "ollama/qwen3-embedding:0.6b", model: "qwen3-embedding:0.6b", dimensions: 1024 },
vector: {
collections: {
reference: "catalog-workspace-reference",
memory: "catalog-workspace-memory",
},
dimensions: 1024,
distance: "cosine",
},
embedding: {
id: "ollama/qwen3-embedding:0.6b",
model: "qwen3-embedding:0.6b",
dimensions: 1024,
},
roots: { artifacts: "/data/artifacts", indexes: "/data/indexes" },
},
effectiveConfigIdentity: "workspace://psd-clinical@v1:" + "d".repeat(64),
effectiveConfigIdentity: "workspace://catalog-workspace@v1:" + "d".repeat(64),
configFingerprint: "sha256:" + "e".repeat(64),
inputFingerprint: "sha256:" + "f".repeat(64),
release: () => undefined,
@@ -112,385 +73,208 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
};
}
function fixture(workspace = baseWorkspace) {
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
roots.push(dataRoot);
const requests: ChildProcessRequest[] = [];
const runChild = vi.fn(async (request: ChildProcessRequest) => {
requests.push(request);
return { exitCode: 0, stdout: JSON.stringify({ status: "succeeded" }), stderr: "" };
});
const service = new WorkspacePreprocessingService({
dataRoot,
acquireActiveRuntime: async () => runtime(workspace),
runChild,
listSessions: async () => [],
semanticPreflight: async () => ({ ok: true }),
evidencePreflight: async () => ({ ok: true }),
});
return { dataRoot, runChild, requests, service };
const database = {
id: "database-1",
workspaceId: "catalog-workspace",
engine: "postgres",
databaseName: "warehouse",
schema: "analytics",
version: 4,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
connectionStatus: "reachable",
schemaSyncedVersion: 4,
metadataContentRevision: 9,
preprocessingStatus: "running",
} as const;
function repository(overrides: Record<string, unknown> = {}) {
const finishPreprocessing = vi.fn(async () => ({
...database,
preprocessingStatus: "succeeded" as const,
preprocessedMetadataRevision: 9,
}));
return {
beginPreprocessing: vi.fn(async () => ({ kind: "started" as const, database })),
finishPreprocessing,
clearPreprocessing: vi.fn(async () => ({ kind: "cleared" as const, database })),
getByWorkspace: vi.fn(async () => database),
listTables: vi.fn(async () => [{
id: "table-1", databaseId: database.id, name: "orders",
description: "Curated orders", generatedDescription: "Generated orders",
sourceComment: "PostgreSQL orders", version: 1,
createdAt: database.createdAt, updatedAt: database.updatedAt,
lastSyncedDatabaseVersion: 4, lastSyncedAt: database.updatedAt,
}]),
listColumns: vi.fn(async () => [{
id: "column-1", tableId: "table-1", name: "customer_id", ordinalPosition: 1,
dataType: "uuid", isNullable: false, defaultExpression: null,
primaryKeyPosition: null, isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1,
sourceComment: "PostgreSQL customer", description: null,
generatedDescription: "Generated customer", sensitive: true,
sensitivityReason: "identifier", lastSyncedDatabaseVersion: 4,
lastSyncedAt: database.updatedAt, version: 1,
createdAt: database.createdAt, updatedAt: database.updatedAt,
}]),
listRelationships: vi.fn(async () => []),
listLogicalRelationships: vi.fn(async () => []),
...overrides,
} as any;
}
test("preprocess dwh uses fixed argv and resumes outer state without rerunning a completed stage", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
stderr: "",
});
const first = await f.service.preprocessDwh({ workspaceId: "psd-clinical" });
expect(first).toMatchObject({
status: "succeeded",
code: "ok",
operation: "preprocess dwh",
completedStages: ["dwh"],
childRuns: { dwh: "d".repeat(32) },
});
expect((f.runChild.mock.calls[0]![0] as ChildProcessRequest).argv).toEqual([
"preprocess", "dwh", "--steps", "introspect,lsh", "--json", "-c", "/dev/fd/3",
]);
const second = await f.service.preprocessDwh({
workspaceId: "psd-clinical",
resumeRunId: first.runId!,
});
expect(second.status).toBe("unchanged");
expect(f.runChild).toHaveBeenCalledTimes(1);
});
test("schema suggest-fks publishes a candidate artifact and blocks full runs for manual review", async () => {
const f = fixture();
f.runChild
.mockResolvedValueOnce({
function service(options: {
repository?: any;
runChild?: (request: ChildProcessRequest) => Promise<{
exitCode: number; stdout: string; stderr: string;
}>;
semanticPreflight?: () => Promise<
{ ok: true } | { ok: false; code: "semantic_index_incompatible" }
>;
} = {}) {
const dataRoot = mkdtempSync(join(tmpdir(), "tht-catalog-preprocessing-"));
roots.push(dataRoot);
return new WorkspacePreprocessingService({
dataRoot,
acquireActiveRuntime: async () => runtime(),
runChild: options.runChild ?? (async () => ({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: []\n",
}),
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
stderr: "",
})),
semanticPreflight: options.semanticPreflight ?? (async () => ({ ok: true })),
evidencePreflight: async () => ({ ok: true }),
catalogRepository: options.repository,
});
}
test("complete preprocessing snapshots Catalog metadata and commits its PostgreSQL state", async () => {
const catalog = repository();
const runChild = vi.fn(async (request: ChildProcessRequest) => {
const snapshotPath = request.argv[request.argv.indexOf("--catalog-metadata") + 1]!;
const snapshot = JSON.parse(readFileSync(snapshotPath, "utf8"));
expect(snapshot).toMatchObject({
workspaceId: "catalog-workspace",
databaseName: "warehouse",
metadataContentRevision: 9,
tables: [{
name: "orders",
description: "Curated orders",
descriptionSource: "curated",
columns: [{
name: "customer_id",
description: "Generated customer",
descriptionSource: "generated",
sensitive: true,
}],
}],
});
const result = await f.service.run({ workspaceId: "psd-clinical" });
expect(result).toMatchObject({
status: "blocked",
code: "manual_review_required",
completedStages: ["dwh", "fk_suggest"],
});
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv[0])).toEqual(["preprocess", "schema"]);
});
test("schema check requires the exact candidate digest and stages annotations via a temp file without recording a review", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: []\n",
}),
stderr: "",
});
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
await expect(f.service.checkSchema({
workspaceId: "psd-clinical",
annotationsYaml: "tables: {}\n",
reviewedCandidatesDigest: "sha256:" + "f".repeat(64),
})).resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
const reviewedDigest = suggest.artifactIdentities![0]!.digest;
let stagedPath = "";
f.runChild.mockImplementationOnce(async (request: ChildProcessRequest) => {
stagedPath = request.argv[request.argv.indexOf("--annotations") + 1]!;
expect(readFileSync(stagedPath, "utf8")).toBe("tables: {}\n");
expect(request.argv).toEqual([
"schema", "check", "--annotations", stagedPath,
"--reviewed-candidates", reviewedDigest,
"--json", "-c", "/dev/fd/3",
]);
return {
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
orphan_count: 0,
annotations_digest: "sha256:annotations",
reviewed_candidates_digest: reviewedDigest,
}),
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
stderr: "",
};
});
const checked = await f.service.checkSchema({
workspaceId: "psd-clinical",
annotationsYaml: "tables: {}\n",
reviewedCandidatesDigest: reviewedDigest,
});
expect(checked).toMatchObject({ status: "succeeded", code: "ok" });
expect(() => readFileSync(stagedPath, "utf8")).toThrow();
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
expect(state.readFkReview(suggest.runId!)).toBeUndefined();
});
test("index schema fails closed when semantic preflight refuses the collection", async () => {
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
roots.push(dataRoot);
const runChild = vi.fn();
const service = new WorkspacePreprocessingService({
dataRoot,
acquireActiveRuntime: async () => runtime(baseWorkspace),
runChild,
listSessions: async () => [],
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
evidencePreflight: async () => ({ ok: true }),
const result = await service({ repository: catalog, runChild }).run({
workspaceId: "catalog-workspace",
});
const result = await service.indexSchema({ workspaceId: "psd-clinical" });
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
expect(runChild).not.toHaveBeenCalled();
});
test("filesystem Evidence proceeds after materialization and private HTTP hosts outside the allowlist are refused", async () => {
const filesystem = fixture(filesystemWorkspace);
filesystem.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1 } }),
stderr: "",
});
const materialized = await filesystem.service.preprocessEvidence({ workspaceId: "fs-workspace" });
expect(materialized).toMatchObject({ status: "succeeded", code: "ok" });
expect(filesystem.runChild).toHaveBeenCalledTimes(1);
const httpDataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
roots.push(httpDataRoot);
const httpService = new WorkspacePreprocessingService({
dataRoot: httpDataRoot,
acquireActiveRuntime: async () => runtime(privateHttpWorkspace, "http-workspace"),
runChild: vi.fn(),
listSessions: async () => [],
semanticPreflight: async () => ({ ok: true }),
evidencePreflight: async () => ({ ok: true }),
httpPrivateHostAllowlist: ["metadata.internal"],
});
const refused = await httpService.preprocessEvidence({ workspaceId: "http-workspace" });
expect(refused).toMatchObject({ status: "failed", code: "egress_policy_refused" });
});
test("full runs follow the explicit order and finish unchanged when no Evidence source exists", async () => {
const f = fixture();
f.runChild
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 0,
candidate_digest: "sha256:" + "0".repeat(64),
candidate_yaml: "tables: []\n",
}),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 },
}),
stderr: "",
});
const result = await f.service.run({ workspaceId: "psd-clinical" });
expect(result).toMatchObject({
status: "succeeded",
code: "ok",
completedStages: ["dwh", "fk_suggest", "schema_index"],
warnings: ["workspace has no Evidence source"],
completedStages: ["catalog_snapshot", "catalog_metadata", "lsh", "schema_index"],
});
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv.slice(0, 2).join(" "))).toEqual([
"preprocess dwh",
"schema suggest-fks",
"vector index-schema",
]);
expect(runChild).toHaveBeenCalledWith(expect.objectContaining({
argv: [
"preprocess", "catalog", "--catalog-metadata",
expect.stringMatching(/\/catalog-metadata\.json$/),
"--json", "-c", "/dev/fd/3",
],
}));
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
"catalog-workspace",
9,
"sha256:" + "f".repeat(64),
{ status: "succeeded" },
);
});
test("schema accept validates the synced Git blob and records the review", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: {}\n",
}),
stderr: "",
});
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
const runId = suggest.runId!;
const candidateDigest = suggest.artifactIdentities![0]!.digest;
const synced = syncAnnotations({
dataRoot: f.dataRoot,
workspaceId: "psd-clinical",
commit: "a".repeat(40),
blobId: "b".repeat(40),
contents: Buffer.from("tables: {}\n"),
});
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
orphan_count: 0,
annotations_digest: synced.contentDigest,
reviewed_candidates_digest: candidateDigest,
}),
stderr: "",
});
const result = await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
expect(result).toMatchObject({ status: "succeeded", code: "ok", operation: "schema accept" });
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
expect(state.readFkReview(runId)).toMatchObject({
reviewedCandidatesDigest: candidateDigest,
annotationsDigest: synced.contentDigest,
workspaceRevision: "a".repeat(40),
blobId: "b".repeat(40),
});
test("preprocessing fails closed when the PostgreSQL Catalog is unavailable", async () => {
const result = await service().run({ workspaceId: "catalog-workspace" });
expect(result).toMatchObject({ status: "failed", code: "catalog_not_ready" });
});
test("schema accept fails closed without --yes, for an unknown run, or with no synced annotations", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: {}\n",
}),
stderr: "",
test("preprocessing refuses a concurrent Catalog run without touching derived data", async () => {
const catalog = repository({
beginPreprocessing: vi.fn(async () => ({ kind: "already_running" as const })),
});
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
const runId = suggest.runId!;
const runChild = vi.fn();
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: false }))
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
const result = await service({ repository: catalog, runChild }).run({
workspaceId: "catalog-workspace",
});
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId: "e".repeat(32), yes: true }))
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true }))
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
expect(f.runChild).toHaveBeenCalledTimes(1);
expect(result).toMatchObject({ status: "failed", code: "preprocessing_conflict" });
expect(runChild).not.toHaveBeenCalled();
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
});
test("full runs continue after schema accept only when the accepted blob matches the current revision", async () => {
const f = fixture();
const revision = "a".repeat(40);
f.runChild
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
stderr: "",
});
const blocked = await f.service.run({ workspaceId: "psd-clinical" });
expect(blocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
const runId = blocked.runId!;
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
const candidateDigest = state.readFkCandidates(runId)!.digest;
const synced = syncAnnotations({
dataRoot: f.dataRoot,
workspaceId: "psd-clinical",
commit: revision,
blobId: "b".repeat(40),
contents: Buffer.from("tables: {}\n"),
test("preprocessing records a failed PostgreSQL state when its hidden worker fails", async () => {
const catalog = repository();
const instance = service({
repository: catalog,
runChild: async () => ({ exitCode: 1, stdout: "", stderr: "private failure" }),
});
// Accept writes the review; the resume then passes the gate and reaches schema indexing.
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", orphan_count: 0, annotations_digest: synced.contentDigest, reviewed_candidates_digest: candidateDigest }),
stderr: "",
});
await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 } }),
stderr: "",
});
const resumed = await f.service.run({ workspaceId: "psd-clinical", resumeRunId: runId });
expect(resumed).toMatchObject({ status: "succeeded", code: "ok" });
// A review whose accepted blob digest no longer matches the current revision stays blocked.
const second = fixture();
second.runChild
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
stderr: "",
});
const secondBlocked = await second.service.run({ workspaceId: "psd-clinical" });
const secondRunId = secondBlocked.runId!;
const secondState = new PreprocessingStateStore({ dataRoot: second.dataRoot, workspaceId: "psd-clinical" });
const secondCandidate = secondState.readFkCandidates(secondRunId)!.digest;
secondState.writeFkReview(secondRunId, {
reviewedCandidatesDigest: secondCandidate,
annotationsDigest: "sha256:" + "0".repeat(64),
workspaceRevision: revision,
blobId: "b".repeat(40),
});
const stillBlocked = await second.service.run({ workspaceId: "psd-clinical", resumeRunId: secondRunId });
expect(stillBlocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
await expect(instance.run({ workspaceId: "catalog-workspace" })).rejects.toThrow(
"workspace child failed",
);
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
"catalog-workspace",
9,
"sha256:" + "f".repeat(64),
{ status: "failed", errorCode: "schema_index_failed" },
);
});
test("vector rebuild recreates the full collection contract including keyword indexes", async () => {
const f = fixture();
// mock fetch: DELETE ok, then reconcileCollection self-heals create + indexes (real fetch in deps)
const calls: string[] = [];
const fakeFetch = async (url: string, init?: any) => {
calls.push(`${init?.method ?? "GET"} ${url}`);
if ((init?.method ?? "GET") === "DELETE") return new Response("", { status: 200 });
if (url.endsWith("/collections/psd-clinical") && init?.method === "PUT") return new Response("", { status: 200 });
if (url.endsWith("/collections/psd-clinical") && init?.method === "GET") {
return new Response(JSON.stringify({ result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, payload_schema: { content_hash: { data_type: "keyword" }, document_id: { data_type: "keyword" }, kind: { data_type: "keyword" }, record_key: { data_type: "keyword" }, record_kind: { data_type: "keyword" }, vector_generation: { data_type: "keyword" }, workspace_id: { data_type: "keyword" }, workspace_revision: { data_type: "keyword" } } } }), { status: 200 });
}
if (url.endsWith("/collections/psd-clinical/index") && init?.method === "PUT") return new Response("", { status: 200 });
return new Response(JSON.stringify({ result: {} }), { status: 200 });
};
const service = new WorkspacePreprocessingService({
dataRoot: f.dataRoot,
acquireActiveRuntime: async () => runtime(baseWorkspace),
runChild: vi.fn(),
listSessions: async () => [],
semanticPreflight: async () => ({ ok: true }),
evidencePreflight: async () => ({ ok: true }),
test("semantic preflight failure is persisted before returning", async () => {
const catalog = repository();
const result = await service({
repository: catalog,
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
}).run({ workspaceId: "catalog-workspace" });
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
"catalog-workspace",
9,
"sha256:" + "f".repeat(64),
{ status: "failed", errorCode: "semantic_index_incompatible" },
);
});
test("clear invalidates Catalog readiness before clearing only derived worker data", async () => {
const catalog = repository();
const runChild = vi.fn(async () => ({
exitCode: 0,
stdout: JSON.stringify({ counts: { referenceCollections: 1, derivedPaths: 3 } }),
stderr: "",
}));
const result = await service({ repository: catalog, runChild }).clear({
workspaceId: "catalog-workspace",
});
// replace global fetch used by vectorRebuild/reconcileCollection
const original = globalThis.fetch;
globalThis.fetch = fakeFetch as any;
try {
const result = await service.vectorRebuild({ workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true });
expect(result).toMatchObject({ status: "succeeded", code: "ok" });
} finally {
globalThis.fetch = original;
}
expect(calls.some((c) => c.startsWith("DELETE "))).toBe(true);
expect(catalog.clearPreprocessing).toHaveBeenCalledWith("catalog-workspace");
expect(runChild).toHaveBeenCalledWith({
argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
configPath: expect.any(String),
});
expect(result).toMatchObject({
status: "succeeded",
code: "ok",
operation: "preprocess clear",
counts: { referenceCollections: 1, derivedPaths: 3 },
});
});
backend/test/workspace-preprocessing-state.test.ts
+9
View File
@@ -47,6 +47,15 @@ test("job state creates durable 0600 JSON and enforces same-revision resume", as
embeddingId: "ollama/qwen3-embedding:0.6b",
embeddingDimensions: 1024,
});
expect(store.readLatestJob()).toMatchObject({
runId: job.runId,
status: "active",
completedStages: [],
});
job.completedStages.push("catalog_snapshot");
store.writeJob(job);
expect(store.readLatestJob()?.completedStages).toEqual(["catalog_snapshot"]);
await expect(store.beginJob({
operation: "preprocess dwh",
backend/test/workspace-registry.test.ts
+2 -131
View File
@@ -19,11 +19,6 @@ const validYaml = `workspace:
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
`;
function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
@@ -34,119 +29,12 @@ function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
`);
}
function withDwhRestTransport(source: string): string {
return source.replace(
"supported_transports: [postgres_direct]",
"supported_transports: [postgres_direct, rest_api]",
);
}
function withDwhRestDiagnostic(source: string): string {
return withDwhRestTransport(source).concat(`diagnostics:
dwh_rest:
method: GET
path: /health
auth: none
response:
database: database
schema: schema
`);
}
function withEmbeddingDiagnostic(source: string): string {
return source.concat(`diagnostics:
embedding:
method: GET
path: /models
auth: none
response:
model: model
dimensions: dimensions
`);
}
function withDwhRestAndEmbeddingDiagnostics(source: string): string {
return withDwhRestTransport(source).concat(`diagnostics:
dwh_rest:
method: GET
path: /health
auth: none
response:
database: database
schema: schema
embedding:
method: GET
path: /models
auth: none
response:
model: model
dimensions: dimensions
`);
}
function withVectorRestTransport(source: string): string {
return source.replace(
"supported_transports: [pgvector_direct]",
"supported_transports: [pgvector_direct, rest_api]",
);
}
function withVectorMetadataDiagnostic(source: string): string {
return withVectorRestTransport(source).concat(`diagnostics:
vector_rest:
metadata:
method: GET
path: /metadata
auth: none
response:
collection: collection
dimensions: dimensions
distance: distance
`);
}
function withReversibleVectorProbe(source: string): string {
return withVectorRestTransport(source).concat(`diagnostics:
vector_rest:
metadata:
method: GET
path: /metadata
auth: none
response:
collection: collection
dimensions: dimensions
distance: distance
reversible_probe:
method: POST
path: /probe
auth: bearer
response:
operation: operation
`);
}
function legacyV1Yaml(source = validYaml): string {
return source
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
.replace(" dimensions: 1024", " dimensions: 768")
.replace(" provider: ollama_internal", " provider: ollama_compatible")
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
.replace(" dimensions: 1024", " dimensions: 768")
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
.replace("schema_version: 4", "schema_version: 1");
return source.replace("schema_version: 4", "schema_version: 1");
}
function legacyV2Yaml(source = validYaml): string {
return source
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
.replace(" dimensions: 1024", " dimensions: 768")
.replace(" provider: ollama_internal", " provider: ollama_compatible")
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
.replace(" dimensions: 1024", " dimensions: 768")
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
.replace("schema_version: 4", "schema_version: 2");
return source.replace("schema_version: 4", "schema_version: 2");
}
const runFile = promisify(execFile);
@@ -605,23 +493,6 @@ test("keeps content-only historical descriptor revisions distinguishable by comm
expect(oldPinned.workspace).toEqual(newPinned.workspace);
});
test.each([
["adds", validYaml, withDwhRestDiagnostic(validYaml)],
["removes", withDwhRestDiagnostic(validYaml), validYaml],
])("pulls a curator change that %s a diagnostics branch without API rewrite", async (_operation, baseSource, remoteSource) => {
const remote = await fixture(baseSource);
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
await registry.bootstrap();
const initial = await registry.read("psd-clinical");
writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), remoteSource);
await git(remote.source, ["add", "psd-clinical/workspace.yaml"]);
await git(remote.source, ["commit", "-m", `Registry ${_operation} diagnostic branch`]);
await git(remote.source, ["push", "origin", "main"]);
await registry.pull();
const updated = await registry.read("psd-clinical");
expect(updated.revision.commit).not.toBe(initial.revision.commit);
});
test.each([
["v1", legacyV1Yaml()],
["v2", legacyV2Yaml()],
backend/test/workspace-runtime-config-lease.test.ts
+55 -37
View File
@@ -1,6 +1,5 @@
import { execFile } from "node:child_process";
import {
chmodSync,
existsSync,
mkdtempSync,
mkdirSync,
@@ -27,6 +26,7 @@ import {
renderActiveWorkspaceRuntime,
renderWorkspaceRuntimeFromSnapshotPath,
} from "../src/workspaces/runtime-config-lease.js";
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
const runFile = promisify(execFile);
const roots: string[] = [];
@@ -70,11 +70,6 @@ workspaces: [{id: psd-clinical, name: Runtime Lease}]
id: psd-clinical
name: Runtime Lease
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: filesystem
@@ -88,9 +83,6 @@ evidence:
await git(source, ["push", "origin", "main"]);
mkdirSync(secretRoot);
const passwordFile = join(secretRoot, "dwh-password");
writeFileSync(passwordFile, "secret", { mode: 0o600 });
chmodSync(passwordFile, 0o600);
mkdirSync(dataRoot);
const registryConfig: WorkspaceRegistryConfig = {
@@ -107,12 +99,31 @@ evidence:
const registry = new WorkspaceRegistry(registryConfig);
await registry.bootstrap();
const revision = (await registry.list())[0];
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse.internal");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "reader");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE", passwordFile);
const workspaceSecretStore = new WorkspaceSecretStore({
root: join(root, "vault"),
runtimeRoot: join(root, "runtime-secrets"),
installationId: "test",
});
workspaceSecretStore.putMany("psd-clinical", { "catalog.dwh.password": "secret" });
const catalogDatabase = {
id: "database-1",
workspaceId: "psd-clinical",
engine: "postgres" as const,
databaseName: "analytics",
schema: "mart",
binding: {
transport: "postgres_direct" as const,
host: "warehouse.internal",
port: 5432,
username: "reader",
},
version: 1,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
connectionStatus: "reachable" as const,
metadataContentRevision: 1,
preprocessingStatus: "failed" as const,
};
return {
dataRoot,
@@ -121,6 +132,8 @@ evidence:
registry,
registryConfig,
revision,
workspaceSecretStore,
catalogDatabase,
};
}
@@ -140,6 +153,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const active = await renderActiveWorkspaceRuntime({
workspaceId: "psd-clinical",
@@ -150,6 +165,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
expect(active.renderedConfig).toBe(direct.renderedConfig);
@@ -158,27 +175,6 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/);
});
test("renders a revision-qualified annotations root for the active revision", async () => {
const f = await fixture();
const active = await renderActiveWorkspaceRuntime({
workspaceId: "psd-clinical",
registry: f.registry,
registryConfig: f.registryConfig,
harnessDir: f.harnessDir,
configPath: "config/tht.yaml",
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
});
const rendered = parse(active.renderedConfig) as Record<string, any>;
expect(rendered.paths.annotations_root).toBe(
join(f.dataRoot, "sessions", "psd-clinical", "revisions", f.revision.commit, "artifacts"),
);
expect(rendered.roots.annotations_root).toBe(rendered.paths.annotations_root);
expect(rendered.paths.artifacts).toBe(join(f.dataRoot, "sessions", "psd-clinical", "artifacts"));
});
test("deterministic operator leases are keyed by logical identity and stable across calls", async () => {
const f = await fixture();
const first = await publishDeterministicRuntimeConfigLease({
@@ -190,6 +186,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const second = await publishDeterministicRuntimeConfigLease({
workspaceId: "psd-clinical",
@@ -200,6 +198,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const suffix = first.inputFingerprint.slice(7, 23);
@@ -236,7 +236,11 @@ test("deterministic operator leases are keyed by logical identity and stable acr
path: first.path,
});
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal");
const changedDatabase = {
...f.catalogDatabase,
binding: { ...f.catalogDatabase.binding, host: "warehouse-two.internal" },
version: 2,
};
const changed = await publishDeterministicRuntimeConfigLease({
workspaceId: "psd-clinical",
registry: f.registry,
@@ -246,6 +250,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: changedDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
expect(changed.path).not.toBe(first.path);
expect(changed.inputFingerprint).not.toBe(first.inputFingerprint);
@@ -267,6 +273,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
})).toThrow(/trusted runtime snapshot/i);
expect(() => renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: symlink,
@@ -275,6 +283,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
})).toThrow(/trusted runtime snapshot/i);
});
@@ -288,6 +298,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const lease = await publishDeterministicRuntimeConfigLease({
workspaceId: "psd-clinical",
@@ -298,6 +310,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const sessionCanonical = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(parse(session.renderedConfig)));
@@ -319,6 +333,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const firstIdentity = firstLease.effectiveConfigIdentity;
@@ -341,6 +357,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
expect(secondLease.workspaceRevision).toBe(current.commit);
backend/test/workspace-runtime-handoff.test.ts
+53 -39
View File
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import {
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync,
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
@@ -25,11 +25,6 @@ const canonicalWorkspace = `workspace:
id: psd-clinical
name: Runtime handoff
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
`;
const filesystemWorkspace = `${canonicalWorkspace}evidence:
@@ -106,6 +101,38 @@ async function fixture(workspaceSource = filesystemWorkspace) {
const registry = new WorkspaceRegistry(registryConfig);
await registry.bootstrap();
const revision = (await registry.list())[0];
const workspaceSecretStore = new WorkspaceSecretStore({
root: join(root, "workspace-secrets"),
runtimeRoot: join(root, "workspace-secret-runtime"),
installationId: "test",
});
workspaceSecretStore.putMany("psd-clinical", {
"catalog.dwh.password": "dwh-password-value",
"evidence.signed_urls": secretContents["evidence-signed-urls.json"],
"evidence.access_key": secretContents["evidence-access"],
"evidence.secret_key": secretContents["evidence-secret"],
"evidence.session_token": secretContents["evidence-token"],
});
const catalogDatabase = {
id: "database-1",
workspaceId: "psd-clinical",
engine: "postgres" as const,
databaseName: "analytics",
schema: "mart",
binding: {
transport: "postgres_direct" as const,
host: "dwh.invalid",
port: 5432,
username: "reader",
},
version: 1,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
connectionStatus: "reachable" as const,
metadataContentRevision: 1,
preprocessingStatus: "failed" as const,
};
const catalogRepository = { getByWorkspace: async () => catalogDatabase } as any;
const environment = {
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.invalid",
@@ -119,7 +146,10 @@ async function fixture(workspaceSource = filesystemWorkspace) {
};
for (const [name, value] of Object.entries(environment)) vi.stubEnv(name, value);
vi.stubEnv("THT_HOME", join(root, "home"));
return { root, source, dataRoot, secretRoot, registry, registryConfig, revision };
return {
root, source, dataRoot, secretRoot, registry, registryConfig, revision,
workspaceSecretStore, catalogDatabase, catalogRepository,
};
}
function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
@@ -130,6 +160,8 @@ function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
dataRoot: f.dataRoot,
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
secretRoots: f.registryConfig.secretRoots,
workspaceSecretStore: f.workspaceSecretStore,
catalogRepository: f.catalogRepository,
} as any);
}
@@ -165,7 +197,7 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
runtimeRoot,
installationId: "test",
});
secretStore.put("psd-clinical", "dwh.password", "vault-runtime-password");
secretStore.put("psd-clinical", "catalog.dwh.password", "vault-runtime-password");
const runner = new ThtRunner({
thtBin,
harnessDir,
@@ -174,9 +206,10 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
secretRoots: f.registryConfig.secretRoots,
workspaceSecretStore: secretStore,
catalogRepository: f.catalogRepository,
} as any);
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const rendered = parse(readFileSync(lease.path, "utf8")) as {
database: { password_file: string };
};
@@ -185,30 +218,11 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
expect(existsSync(rendered.database.password_file)).toBe(false);
});
test("ThtRunner binds and cleans the effective relationship snapshot with its runtime lease", async () => {
const f = await fixture();
const runner = runnerFor(f);
const relationships = JSON.stringify({
schemaVersion: 1,
workspaceId: "psd-clinical",
relationships: [],
});
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath, relationships);
const rendered = parse(readFileSync(lease.path, "utf8")) as {
paths: { effective_relationships: string };
};
expect(readFileSync(rendered.paths.effective_relationships, "utf8")).toBe(relationships);
lease.release();
expect(existsSync(rendered.paths.effective_relationships)).toBe(false);
});
test("separate runtime leases hand off byte-identical revision Evidence configs accepted by tht", async () => {
const f = await fixture();
const runner = runnerFor(f);
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const second = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const second = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const expectedRoot = join(
f.registryConfig.root,
"snapshots",
@@ -263,7 +277,7 @@ test("separate runtime leases hand off byte-identical revision Evidence configs
test("real Evidence-content-only commit changes runtime identity and root with identical descriptor YAML", async () => {
const f = await fixture();
const runner = runnerFor(f);
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const descriptorBefore = readFileSync(f.revision.snapshotPath, "utf8");
writeFileSync(
join(f.source, "psd-clinical", "evidence", "guide.md"),
@@ -274,7 +288,7 @@ test("real Evidence-content-only commit changes runtime identity and root with i
await git(f.source, ["push", "origin", "main"]);
await f.registry.pull();
const current = (await f.registry.list())[0];
const second = runner.acquireWorkspaceRuntime(current.snapshotPath);
const second = await runner.acquireWorkspaceRuntime(current.snapshotPath);
try {
expect(current.commit).not.toBe(f.revision.commit);
@@ -317,13 +331,13 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
max_cache_bytes: 67890
`));
const runner = runnerFor(f);
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
try {
const yaml = readFileSync(lease.path, "utf8");
expect(parse(yaml).evidence.sources).toEqual([{
type: "http",
provenance_urls: ["https://evidence.example.test/guide.md"],
signed_urls_file: realpathSync(join(f.secretRoot, "evidence-signed-urls.json")),
signed_urls_file: expect.stringContaining("/workspace-secret-runtime/"),
connect_timeout: 1.25,
read_timeout: 30.001,
max_bytes: 12_345,
@@ -343,7 +357,7 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
}
});
test("static S3 Evidence resolves only configured secret-root file paths", async () => {
test("static S3 Evidence resolves only ephemeral vault materializations", async () => {
const f = await fixture(evidenceWorkspace(` type: s3
uri: s3://clinical-evidence/published/
endpoint_url: https://s3.example.test/
@@ -361,7 +375,7 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
retain_published_generations: 7
`));
const runner = runnerFor(f);
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
try {
const yaml = readFileSync(lease.path, "utf8");
expect(parse(yaml).evidence.sources).toEqual([{
@@ -370,9 +384,9 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
prefix: "published/",
endpoint_url: "https://s3.example.test/",
region: "eu-west-1",
access_key_file: realpathSync(join(f.secretRoot, "evidence-access")),
secret_key_file: realpathSync(join(f.secretRoot, "evidence-secret")),
session_token_file: realpathSync(join(f.secretRoot, "evidence-token")),
access_key_file: expect.stringContaining("/workspace-secret-runtime/"),
secret_key_file: expect.stringContaining("/workspace-secret-runtime/"),
session_token_file: expect.stringContaining("/workspace-secret-runtime/"),
trusted_endpoint: true,
allow_private_endpoint: true,
allow_insecure_endpoint: false,
backend/test/workspace-runtime-renderer.test.ts
+11 -4
View File
@@ -10,9 +10,9 @@ import {
type SemanticRuntimeConfig,
} from "../src/workspaces/runtime-renderer.js";
import { supportsSessionRuntime } from "../src/workspaces/bindings.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const workspaceV4 = parseWorkspaceYaml(`workspace:
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -70,7 +70,14 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu
},
dwh: { type: "postgres_direct" },
resources: {
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
vector: {
engine: "qdrant",
base_url: "http://qdrant:6333",
collections: {
reference: "psd-clinical-reference",
memory: "psd-clinical-memory",
},
},
embeddings: {
provider: "ollama_internal", base_url: "http://embedding:11434",
id: "ollama/qwen3-embedding:0.6b",
@@ -133,7 +140,7 @@ function evidenceWorkspace(
policy?: Record<string, unknown>,
evidenceSchemaVersion?: number,
) {
return parseWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
return parseRuntimeWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
evidenceSchemaVersion === undefined ? "" : `\n schema_version: ${evidenceSchemaVersion}`
}\n source: ${JSON.stringify(source)}${
policy === undefined ? "" : `\n policy: ${JSON.stringify(policy)}`
backend/test/workspace-secret-requirements.test.ts
+2 -2
View File
@@ -8,12 +8,12 @@ import {
resolveRuntimeBindingsWithWorkspaceSecrets,
} from "../src/workspaces/secret-requirements.js";
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const roots: string[] = [];
function workspace(extra = "") {
return parseWorkspaceYaml(`workspace:
return parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
backend/test/workspace-v4-migration.test.ts
+2 -5
View File
@@ -23,14 +23,11 @@ test("strict workspace v4 rejects model-bearing v3 descriptors", () => {
expect(() => parseWorkspaceYaml(legacy)).toThrow();
});
test("v3 to v4 migration removes only model/vector policy and bumps the version", () => {
test("v3 to v4 migration removes database/model policy and bumps the version", () => {
const migrated = migrateWorkspaceV3Yaml(legacy);
const workspace = parseWorkspaceYaml(migrated);
expect(workspace.workspace).toMatchObject({ schema_version: 4, id: "abc" });
expect(migrated).not.toMatch(/semantic_index|llm_policy/);
expect(workspace.dwh).toEqual({
engine: "postgres", database: "warehouse", schema: "public",
supported_transports: ["postgres_direct"],
});
expect(workspace.dwh).toBeUndefined();
});
backend/test/workspaces-bindings.test.ts
+5 -5
View File
@@ -7,9 +7,9 @@ import { afterEach, expect, test } from "vitest";
import {
resolveBinding, resolveEvidenceBinding, resolveRuntimeBindings, supportsSessionRuntime,
} from "../src/workspaces/bindings.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const workspaceV4 = parseWorkspaceYaml(`workspace:
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -63,7 +63,7 @@ test("requires workspace-v4 REST credentials unless the DWH diagnostic declares
THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
}, []).missing).toContain("THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE");
const noAuth = parseWorkspaceYaml(`workspace:
const noAuth = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: No auth
@@ -89,7 +89,7 @@ diagnostics:
test("rejects unsupported transports and secret paths outside configured roots", () => {
const outside = secretPath("outside-password");
const allowed = secretPath("allowed-password");
const directOnly = parseWorkspaceYaml(`workspace:
const directOnly = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Direct only
@@ -130,7 +130,7 @@ test("runtime bindings contain only DWH and Evidence roles", () => {
});
function withEvidence(source: Record<string, unknown>) {
return parseWorkspaceYaml(`workspace:
return parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
backend/test/workspaces-catalog.test.ts
+2 -2
View File
@@ -1,12 +1,12 @@
import { expect, test } from "vitest";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
import {
CATALOG_PATH,
assertCatalogMatchesDescriptor,
parseWorkspaceCatalogYaml,
} from "../src/workspaces/catalog.js";
const descriptor = parseWorkspaceYaml(`workspace:
const descriptor = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd
name: Policlinico San Donato
backend/test/workspaces-contracts.test.ts
+9 -9
View File
@@ -3,9 +3,9 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { buildInstallationContract, renderWorkspaceDocs } from "../src/workspaces/contracts.js";
import { type CanonicalWorkspace, parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { type CanonicalWorkspace, parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const workspaceV4 = parseWorkspaceYaml(`workspace:
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -38,7 +38,7 @@ test.each([
["rest_api", "BASE_URL", "HOST"],
["ssh_tunnel", "SSH_PRIVATE_KEY_FILE", "BASE_URL"],
] as const)("documents only the DWH fields for %s", (transport, included, excluded) => {
const descriptor = parseWorkspaceYaml(renderWorkspaceWithoutEvidence()
const descriptor = parseRuntimeWorkspaceYaml(renderWorkspaceWithoutEvidence()
.replace("[postgres_direct]", `[${transport}]`));
const variables = buildInstallationContract(descriptor).variables;
expect(variables.find(({ suffix }) => suffix === included)?.transports).toEqual([transport]);
@@ -87,7 +87,7 @@ test.each([
],
},
])("generates source-specific $mode Evidence file bindings", ({ source, expected }) => {
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
const contract = buildInstallationContract(descriptor);
const evidence = contract.variables.filter((variable) => variable.role === "EVIDENCE");
@@ -102,7 +102,7 @@ test.each([
{ type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" },
{ type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" },
])("omits Evidence installation variables for $type modes without file credentials", (source) => {
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
expect(buildInstallationContract(descriptor).variables.some((variable) => variable.role === "EVIDENCE"))
.toBe(false);
});
@@ -151,7 +151,7 @@ const evidenceSources = [
] as const;
test.each(evidenceSources)("renders deterministic public Evidence docs for $label", ({ source, variables }) => {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
);
const firstContract = buildInstallationContract(descriptor);
@@ -178,7 +178,7 @@ test.each(evidenceSources)("renders deterministic public Evidence docs for $labe
});
test("documents the S3 session token file as optional", () => {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:
source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files }
`,
@@ -197,7 +197,7 @@ test("documents the S3 session token file as optional", () => {
});
test("documents same-revision filesystem ownership without claiming P1 materialization", () => {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:\n source: { type: filesystem, uri: psd-clinical/evidence }\n`,
);
const docs = renderWorkspaceDocs(descriptor).markdown;
@@ -236,7 +236,7 @@ test.each([
const previous = process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE;
process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = binding;
try {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
);
const generated = JSON.stringify({
backend/test/workspaces-diagnostics.test.ts
+15 -10
View File
@@ -9,9 +9,9 @@ import {
type DiagnosticAdapters,
} from "../src/workspaces/diagnostics.js";
import type { RuntimeBindings } from "../src/workspaces/runtime-renderer.js";
import { parseWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
const workspace = parseWorkspaceYaml(`workspace:
const workspace = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -81,9 +81,11 @@ test("diagnoses workspace-v4 DWH plus installation-derived Qdrant and Ollama", a
role: "dwh", transport: "postgres_direct",
resource: { database: "warehouse", schema: "datawarehouse" },
}));
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({
baseUrl: "http://qdrant:6333", collection: "psd-clinical",
}));
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
expect(vi.mocked(adapters.inspectQdrant).mock.calls.map(([request]) => request.collection)).toEqual([
"psd-clinical-reference",
"psd-clinical-memory",
]);
expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({
baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b",
}));
@@ -107,14 +109,14 @@ test("can delegate the DWH probe to Database Management", async () => {
}],
});
expect(adapters.probeConnector).not.toHaveBeenCalled();
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(1);
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
expect(adapters.probeEmbedding).toHaveBeenCalledTimes(1);
});
test("reports incompatible internal Qdrant or Ollama metadata", async () => {
const vector = await diagnose(successfulAdapters({
inspectQdrant: vi.fn(async () => ({
collection: "psd-clinical", dimensions: 768, distance: "cosine",
inspectQdrant: vi.fn(async (request) => ({
collection: request.collection, dimensions: 768, distance: "cosine",
})),
}))(workspace, bindings, { writeProbe: false });
expect(vector.activatable).toBe(false);
@@ -163,7 +165,7 @@ test("keeps workspace-v4 DWH SSH diagnostic-only and runtime-inactive", async ()
});
test("uses the workspace-v4 declared DWH REST diagnostic and auth policy", async () => {
const restWorkspace = parseWorkspaceYaml(`workspace:
const restWorkspace = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: REST workspace
@@ -428,7 +430,7 @@ test("uses a REST secret only as a header and redacts it from failed diagnostics
const credentialFile = join(root, "api-key");
const canary = "CANARY-REST-AUTH-SECRET";
await writeFile(credentialFile, canary);
const restDescriptor = parseWorkspaceYaml(`workspace:
const restDescriptor = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: REST auth
@@ -548,6 +550,9 @@ test("returns observed normalized Qdrant distance for semantic mismatch classifi
test("classifies an observed non-cosine Qdrant distance as semantic incompatibility", async () => {
const fetchMock = vi.fn()
.mockResolvedValueOnce(new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
}), { status: 200 }))
.mockResolvedValueOnce(new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
}), { status: 200 }))
backend/test/workspaces-git-repository.test.ts
-5
View File
@@ -16,11 +16,6 @@ const validYaml = `workspace:
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
`;
const runFile = promisify(execFile);
backend/test/workspaces-schema.test.ts
+10 -23
View File
@@ -15,16 +15,6 @@ export const validYaml = `workspace:
name: Policlinico San Donato
description: Clinical data warehouse workspace
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
port: 5432
timeout_ms: 5000
supported_transports:
- postgres_direct
- rest_api
- ssh_tunnel
`;
test("rejects unknown keys and invalid immutable IDs", () => {
@@ -41,21 +31,18 @@ test("rejects executable or otherwise custom YAML tags in canonical descriptors"
))).toThrow(/tag|yaml/i);
});
test("accepts optional connection ports and timeouts but rejects unsafe values", () => {
expect(parseWorkspaceYaml(validYaml).dwh.port).toBe(5432);
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 0")))
.toThrow(/port/i);
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 65536")))
.toThrow(/port/i);
expect(() => parseWorkspaceYaml(validYaml.replace("timeout_ms: 5000", "timeout_ms: 0")))
.toThrow(/timeout/i);
test("rejects database configuration and diagnostics in authored workspace YAML", () => {
expect(() => parseWorkspaceYaml(`${validYaml}dwh:\n engine: postgres\n database: d\n schema: s\n supported_transports: [postgres_direct]\n`))
.toThrow(/must not contain database configuration/i);
expect(() => parseWorkspaceYaml(`${validYaml}diagnostics:\n dwh_rest:\n method: GET\n path: \/health\n auth: none\n`))
.toThrow(/must not contain database configuration/i);
});
test("accepts a model-free schema v4 workspace", () => {
test("accepts a database-free schema v4 workspace", () => {
expect(parseWorkspaceYaml(validYaml)).toMatchObject({
workspace: { schema_version: 4, id: "psd-clinical" },
dwh: { database: "postgres", schema: "datawarehouse" },
});
expect(parseWorkspaceYaml(validYaml)).not.toHaveProperty("dwh");
});
test("committed example descriptors parse as exact schema v4 workspaces", () => {
@@ -137,7 +124,7 @@ llm_policy:
- zai/glm-5.2
`],
])("rejects schema %s descriptors at parser and object-validator boundaries", (_version, yaml) => {
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|4/i);
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|database configuration|4/i);
expect(() => validateWorkspaceDescriptor(parse(yaml))).toThrow(/schema_version|invalid literal|4/i);
});
@@ -159,7 +146,7 @@ test("constructs diagnostic URLs only when the resolved URL remains on the servi
)).toThrow(/origin/i);
});
test("rejects REST diagnostic declarations without their matching connector transport", () => {
test("rejects REST diagnostic declarations in authored descriptors", () => {
const diagnostics = `diagnostics:
dwh_rest:
method: POST
@@ -171,7 +158,7 @@ test("rejects REST diagnostic declarations without their matching connector tran
`;
const declared = `${validYaml}${diagnostics}`;
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", ""))).toThrow(/dwh_rest/i);
expect(() => parseWorkspaceYaml(declared)).toThrow(/database configuration/i);
});
test("serializes canonical YAML that parses back to the same workspace", () => {
backend/test/workspaces/evidence/preprocessing.test.ts
+24
View File
@@ -138,6 +138,30 @@ test("projects aggregate no-Evidence and completed-stage outcomes without rerunn
expect(deps.persistJob).not.toHaveBeenCalled();
});
test("complete preprocessing preflights BM25 before continuing with Evidence", async () => {
const deps = dependencies();
deps.evidencePreflight.mockResolvedValue({
ok: false as const,
code: "semantic_index_incompatible" as const,
});
const state = job({ completedStages: ["catalog_snapshot", "schema_index"] });
const result = await continueEvidencePreprocessing(
{ evidence: filesystemEvidence, job: state },
deps,
);
expect(deps.evidencePreflight).toHaveBeenCalledOnce();
expect(deps.runStage).not.toHaveBeenCalled();
expect(result).toEqual({
status: "failed",
code: "semantic_index_incompatible",
runId: "a".repeat(32),
childRuns: {},
completedStages: ["catalog_snapshot", "schema_index"],
});
});
test("preserves the narrow standalone projection for an already completed Evidence stage", async () => {
const deps = dependencies();