feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s

This commit is contained in:
Codex
2026-09-06 17:49:35 +02:00
parent 8707ae1d46
commit cffa60772e
141 changed files with 5898 additions and 3015 deletions
+6 -14
View File
@@ -97,31 +97,23 @@ const fleetSnapshot: ObservedSchemaSnapshot = {
}],
};
test("lists every YAML workspace and creates its one database configuration", async () => {
test("lists every workspace and creates its Catalog database configuration", async () => {
const { app, secretStore } = setup();
const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(initial.statusCode).toBe(200);
expect(initial.json()).toMatchObject([{
workspaceId: "psd-clinical",
configured: false,
databaseName: "warehouse",
databaseName: "",
workspaceRevision: { commit: revision.commit, blob: revision.blob },
workspaceEvidence: { sourceType: null, state: "not_declared" },
runtimeBinding: {
transport: "postgres_direct",
configurationState: "configuration_required",
sessionTransportSupported: true,
},
runtimeBinding: null,
}]);
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" });
const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(runtimeReady.json()).toMatchObject([{
runtimeBinding: { configurationState: "ready", sessionTransportSupported: true },
runtimeBinding: null,
}]);
const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
@@ -166,7 +158,7 @@ test("projects remote Evidence credential state without conflating catalog secre
}]);
});
test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
test("lists orphaned records and keeps the REST diagnostic path in the Catalog", async () => {
const { app, repository } = setup();
await repository.create({
workspaceId: "removed-workspace",
@@ -186,7 +178,7 @@ test("lists orphaned records and takes the REST diagnostic path from workspace Y
},
});
expect(created.statusCode).toBe(201);
expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
expect(created.json()).toMatchObject({ binding: { restPath: "/client-controlled" } });
const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
expect(rows).toEqual(expect.arrayContaining([
@@ -577,7 +577,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
expect.objectContaining({
sequence: 3,
level: "info",
message: `Generated description for Catalog Column ${column.id}.`,
message: 'Generated description for Column "patients.birth_date".',
}),
expect.objectContaining({ sequence: 4, level: "info", message: "Description generation completed." }),
]);
@@ -934,7 +934,10 @@ test("generates selected Catalog Columns in caller order through sequential batc
const events = await repository.listDescriptionGenerationEvents(run.id);
expect(events.map((event) => event.sequence)).toEqual(Array.from({ length: 14 }, (_, index) => index + 1));
expect(events.slice(2, -1).map((event) => event.message)).toEqual(
orderedIds.map((targetId) => `Generated description for Catalog Column ${targetId}.`),
orderedIds.map((targetId) => {
const target = columns.find((column) => column.id === targetId)!;
return `Generated description for Column "patients.${target.name}".`;
}),
);
} finally {
pending[0]!.resolve(responseFor(orderedIds.slice(0, 10), 0));
@@ -1803,7 +1806,9 @@ test("retains completed batch writes when a later batch response is malformed",
});
const events = await repository.listDescriptionGenerationEvents(run.id);
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
originalColumns.slice(0, 10).map(
(target) => `Generated description for Column "patients.${target.name}".`,
),
);
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
expect.objectContaining({
@@ -1812,7 +1817,7 @@ test("retains completed batch writes when a later batch response is malformed",
);
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
message: `The model response did not match the required schema. Affected target: Column "patients.${originalColumns[10]!.name}".`,
}));
} finally {
await app.close();
@@ -2301,7 +2306,7 @@ test("generates selected Catalog Tables with structural column context and local
expect((await repository.listDescriptionGenerationEvents(run.id)).map((event) => event.message)).toEqual([
"Description generation queued.",
"Description generation started.",
`Stored non-generatable result for Catalog Table ${table.id}.`,
'Stored non-generatable result for Table "patients".',
"Description generation completed.",
]);
} finally {
@@ -2445,7 +2450,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
expect(`${JSON.stringify(run)}${events.body}`).not.toContain(sensitiveDiagnostic);
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model provider request failed. Affected Catalog Column target: ${column.id}.`,
message: 'The model provider request failed. Affected target: Column "patients.birth_date".',
}));
} finally {
await app.close();
@@ -2536,7 +2541,7 @@ test.each([
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
expect.objectContaining({
level: "error",
message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
message: `${failureMessage} Affected targets: Column "patients.first_column", Column "patients.second_column".`,
}),
);
} finally {
@@ -13,10 +13,12 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
import { loadConfig } from "../src/config.js";
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
@@ -52,10 +54,12 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
await upSensitiveDataFlag(db);
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
await upLogicalRelationships(db);
await upAiTokenUsage(db);
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
await upCatalogPreprocessingState(db);
const repository = new KyselyCatalogRepository(db);
const database = await repository.create({
workspaceId: "psd-clinical",
@@ -282,7 +286,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
expect(events.statusCode).toBe(200);
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
message: 'The model provider request failed. Affected target: Column "patients.status".',
}));
expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
@@ -18,6 +18,7 @@ import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usa
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
@@ -58,6 +59,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
await upCatalogPreprocessingState(db);
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
.select(["engine", "modelId", "policyVersion", "unknown"])
.where("id", "=", historicalSuggestionRunId)
@@ -232,6 +234,77 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
expect(await repository.listSyncRuns(created.id)).toEqual([
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
]);
const lockedDatabase = await repository.create({
workspaceId: "preprocessing-lock",
engine: "postgres",
databaseName: "warehouse",
schema: "public",
binding: {
transport: "postgres_direct", host: "lock.internal", port: 5432, username: "reader",
},
});
await repository.applySchemaSync(
lockedDatabase.id,
lockedDatabase.version,
"all",
[],
{
schemaVersion: 1,
capabilities: { tables: "available", columns: "available", relationships: "available" },
tables: [], columns: [], relationships: [],
},
);
const beforePreprocessing = (await repository.get(lockedDatabase.id))!;
const preprocessing = await repository.beginPreprocessing(
"preprocessing-lock",
"sha256:" + "1".repeat(64),
);
expect(preprocessing).toMatchObject({ kind: "started" });
await expect(db.insertInto("catalogTables").values({
id: randomUUID(),
databaseId: lockedDatabase.id,
name: "blocked_write",
sourceComment: null,
description: null,
generatedDescription: null,
}).execute()).rejects.toThrow("catalog preprocessing is running");
await expect(repository.createDescriptionGenerationRun(
lockedDatabase.id,
"all",
"openai/gpt-5-mini",
"en",
0,
)).rejects.toThrow("catalog preprocessing is running");
await repository.recordTest(lockedDatabase.id, lockedDatabase.version, {
connectionStatus: "reachable",
testedVersion: lockedDatabase.version,
lastTestedAt: "2026-01-01T00:00:00Z",
});
expect((await repository.get(lockedDatabase.id))?.metadataContentRevision)
.toBe(beforePreprocessing.metadataContentRevision);
await expect(repository.finishPreprocessing(
"preprocessing-lock",
beforePreprocessing.metadataContentRevision,
"sha256:" + "1".repeat(64),
{ status: "succeeded" },
)).resolves.toMatchObject({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: beforePreprocessing.metadataContentRevision,
});
await db.insertInto("catalogTables").values({
id: randomUUID(),
databaseId: lockedDatabase.id,
name: "allowed_after_preprocessing",
sourceComment: null,
description: null,
generatedDescription: null,
}).execute();
await expect(repository.get(lockedDatabase.id)).resolves.toMatchObject({
preprocessingStatus: "failed",
metadataContentRevision: beforePreprocessing.metadataContentRevision + 1,
});
expect(await repository.delete(lockedDatabase.id, lockedDatabase.version)).toBe(true);
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
expect(await repository.delete(created.id, 2)).toBe(true);
expect(await repository.list()).toEqual([]);
@@ -355,7 +428,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
}
}, 60_000);
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected and database-wide descriptions", async () => {
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
const db = new Kysely<CatalogDatabase>({
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
@@ -449,6 +522,22 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel
description: "Still curated visits",
generatedDescription: "Generated visits",
});
expect(await repository.consolidateGeneratedDescriptions(
database.id, "database", [],
)).toEqual({ copied: 3, skipped: 1 });
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
description: "Generated visits",
generatedDescription: "Generated visits",
});
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
description: "Generated id",
generatedDescription: "Generated id",
});
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
description: "Keep patient reference",
generatedDescription: null,
});
} finally {
await db.destroy();
await container.stop();
@@ -473,6 +562,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
await upCatalogPreprocessingState(db);
const repository = new KyselyCatalogRepository(db);
const firstDatabase = await repository.create({
workspaceId: "generation-one",
@@ -0,0 +1,105 @@
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test } from "vitest";
import { resolveCatalogRuntimeBinding } from "../src/catalog/runtime-binding.js";
import type { WorkspaceDatabase } from "../src/catalog/types.js";
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
const roots: string[] = [];
afterEach(() => {
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});
test("projects a Catalog database into a runtime without database data in workspace YAML", () => {
const root = mkdtempSync(join(tmpdir(), "thoth-catalog-runtime-"));
roots.push(root);
const secretStore = new WorkspaceSecretStore({
root: join(root, "vault"),
runtimeRoot: join(root, "runtime"),
installationId: "test",
});
secretStore.putMany("sales", {
"catalog.dwh.password": "catalog-password",
"evidence.signed_urls": '["https://signed.example.test/evidence"]',
});
const workspace: WorkspaceDescriptor = {
workspace: {
schema_version: 4,
id: "sales",
name: "Sales",
language: "en",
},
evidence: {
schema_version: 1,
source: {
type: "http",
uris: ["https://evidence.example.test/guide.md"],
authentication: "signed_urls_file",
connect_timeout_ms: 1_000,
read_timeout_ms: 5_000,
max_bytes: 10_000,
max_redirects: 2,
allow_private_hosts: false,
max_cache_bytes: 20_000,
},
policy: { max_chunk_chars: 4_000, retain_published_generations: 1 },
},
};
const database: WorkspaceDatabase = {
id: "database-1",
workspaceId: "sales",
engine: "postgres",
databaseName: "warehouse",
schema: "analytics",
version: 3,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
binding: {
transport: "postgres_direct",
host: "db.internal",
port: 5432,
username: "reader",
},
connectionStatus: "reachable",
metadataContentRevision: 7,
preprocessingStatus: "failed",
};
const lease = resolveCatalogRuntimeBinding({
workspace,
database,
environment: {},
secretRoots: [],
secretStore,
});
const passwordPath = lease.bindings.dwh.values.THT_WS_SALES_DWH_PASSWORD_FILE;
const evidencePath = lease.bindings.evidence.values.THT_WS_SALES_EVIDENCE_SIGNED_URLS_FILE;
try {
expect(workspace.dwh).toBeUndefined();
expect(lease.workspace.dwh).toMatchObject({
database: "warehouse",
schema: "analytics",
supported_transports: ["postgres_direct"],
});
expect(lease.bindings.dwh).toMatchObject({
transport: "postgres_direct",
missing: [],
values: {
THT_WS_SALES_DWH_HOST: "db.internal",
THT_WS_SALES_DWH_PORT: "5432",
THT_WS_SALES_DWH_USER: "reader",
},
});
expect(readFileSync(passwordPath, "utf8")).toBe("catalog-password");
expect(readFileSync(evidencePath, "utf8")).toContain("signed.example.test");
} finally {
lease.release();
}
expect(existsSync(passwordPath)).toBe(false);
expect(existsSync(evidencePath)).toBe(false);
});
+21 -4
View File
@@ -394,12 +394,19 @@ test("consolidates non-empty generated column descriptions and preserves curated
expect(scan).not.toHaveBeenCalled();
});
test("consolidates generated descriptions for every column in a database", async () => {
test("consolidates generated descriptions for every table and column in a database", async () => {
const { app, repository, database, scan } = await setup();
await seedCatalog(repository, database);
const tables = await repository.listTables(database.id);
const patients = tables.find((table) => table.name === "patients")!;
const visits = tables.find((table) => table.name === "visits")!;
await repository.updateTableMetadata(
database.id,
patients.id,
patients.version,
"Curated patients",
"Generated patients",
);
const patientId = (await repository.listColumns(database.id, patients.id))[0]!;
const visitColumns = await repository.listColumns(database.id, visits.id);
const visitId = visitColumns.find((column) => column.name === "id")!;
@@ -432,11 +439,16 @@ test("consolidates generated descriptions for every column in a database", async
const response = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
payload: { target: "database_columns" },
payload: { target: "database" },
});
expect(response.statusCode).toBe(200);
expect(response.json()).toEqual({ copied: 2, skipped: 1 });
expect(response.json()).toEqual({ copied: 3, skipped: 2 });
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
description: "Generated patients",
generatedDescription: "Generated patients",
version: patients.version + 2,
});
expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({
description: "Generated patient identifier",
generatedDescription: "Generated patient identifier",
@@ -529,6 +541,11 @@ test("strictly validates description consolidation database and target ids", asy
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
payload: { target: "tables", targetIds: [table.id], unexpected: true },
}),
app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
payload: { target: "database", targetIds: [table.id] },
}),
app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
@@ -536,7 +553,7 @@ test("strictly validates description consolidation database and target ids", asy
}),
]);
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400]);
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400, 400]);
for (const response of responses) {
expect(response.json()).toEqual({
code: "description_consolidation_invalid",
+17 -5
View File
@@ -41,7 +41,10 @@ function directRendered(): Record<string, unknown> {
vector: {
engine: "qdrant",
base_url: "http://qdrant:6333",
collection: "psd-clinical",
collections: {
reference: "psd-clinical-reference",
memory: "psd-clinical-memory",
},
dimensions: 1024,
distance: "cosine",
collection_lifecycle: "require_existing",
@@ -103,10 +106,10 @@ function restRendered(): Record<string, unknown> {
}
const directCanonical =
`{"schemaVersion":2,"dwh":{` +
`{"schemaVersion":3,"dwh":{` +
`"engine":"postgres","database":"postgres","schema":"datawarehouse",` +
`"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` +
`"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` +
`"vector":{"collections":{"reference":"psd-clinical-reference","memory":"psd-clinical-memory"},"dimensions":1024,"distance":"cosine"},` +
`"embedding":{"id":"ollama/qwen3-embedding:0.6b","model":"qwen3-embedding:0.6b","dimensions":1024},` +
`"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` +
`"indexes":"/data/sessions/psd-clinical/indexes"}}`;
@@ -128,7 +131,7 @@ test("canonical effective config excludes secrets, evidence, session storage, an
expect(json).not.toContain("sources");
expect(json).not.toContain("collection_lifecycle");
expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity
expect(json).not.toContain("memory");
expect(json).not.toContain('"memory":"/data');
expect(json).not.toContain('"sessions"');
});
@@ -190,7 +193,16 @@ test("DWH-affecting changes alter the effective config identity", () => {
const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } };
expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore);
const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record<string, any>).vector, collection: "other" } } };
const changedCollection = {
...base,
resources: {
...base.resources,
vector: {
...(base.resources as Record<string, any>).vector,
collections: { reference: "other-reference", memory: "other-memory" },
},
},
};
expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore);
const changedEmbeddingIdentity = { ...base, resources: { ...base.resources, embeddings: { ...(base.resources as Record<string, any>).embeddings, model: "other-embedding" } } };
@@ -19,11 +19,6 @@ const validYaml = `workspace:
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
`;
async function git(cwd: string, args: string[]): Promise<string> {
-5
View File
@@ -24,11 +24,6 @@ const descriptor = `workspace:
id: research
name: Research
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: filesystem
+90 -13
View File
@@ -12,6 +12,7 @@ import { PiProcessManager } from "../src/pi/pi-process-manager.js";
import { validateDeclarativePiConfig } from "../src/pi/managed-config.js";
import Fastify from "fastify";
import { sessionRoutes } from "../src/routes/sessions.js";
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
const SCRIPT = path.resolve("../harness/tests/fake_pi/scripts/f1_disambiguation.json");
@@ -517,7 +518,7 @@ test("creates a session from the active immutable workspace revision", async ()
}));
});
test("hands one effective relationship snapshot to both retrieval and Pi", async () => {
test("hands one Catalog-backed runtime to both retrieval and Pi", async () => {
const effective = JSON.stringify({
schemaVersion: 1,
workspaceId: "default",
@@ -556,10 +557,9 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
});
expect(response.statusCode).toBe(200);
expect(render).toHaveBeenCalledWith("default");
expect(render).not.toHaveBeenCalled();
expect(acquireWorkspaceRuntime).toHaveBeenCalledWith(
expect.stringContaining("/default.yaml"),
effective,
);
expect(createFor).toHaveBeenCalledWith(
"effective-map",
@@ -574,27 +574,103 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
);
});
test("rejects an SSH-only workspace before persisting or starting a session", async () => {
test("refuses core admission when the workspace preprocessing fingerprint is stale", async () => {
const sessionNew = vi.fn();
const workspaceInputFingerprint = vi.fn(async () => "sha256:current");
const revision = {
id: "default",
commit: "a".repeat(40),
blob: "b".repeat(40),
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/default.yaml`,
};
const catalogRepository = new MemoryCatalogRepository();
const database = await catalogRepository.create({
workspaceId: "default",
engine: "postgres",
databaseName: "warehouse",
schema: "public",
binding: {
transport: "postgres_direct",
host: "db.internal",
port: 5432,
username: "reader",
},
});
await catalogRepository.applySchemaSync(database.id, database.version, "all", [], {
schemaVersion: 1,
capabilities: { tables: "available", columns: "available", relationships: "available" },
tables: [],
columns: [],
relationships: [],
});
const started = await catalogRepository.beginPreprocessing("default", "sha256:previous");
expect(started.kind).toBe("started");
await catalogRepository.finishPreprocessing(
"default",
0,
"sha256:previous",
{ status: "succeeded" },
);
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: { sessionNew, workspaceInputFingerprint } as any,
catalogRepository,
workspaceRegistry: {
acquireSessionRevision: async () => ({
workspace: operationalWorkspace("default"),
revision,
abort: async () => {},
markPersisted: async () => {},
}),
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "default", thinking: "low" }) as any,
});
const response = await app.inject({
method: "POST",
url: "/sessions",
payload: { question: "q", workspaceId: "default" },
});
expect(response.statusCode).toBe(409);
expect(response.json()).toMatchObject({ code: "preprocessing_required" });
expect(workspaceInputFingerprint).toHaveBeenCalledWith(revision.snapshotPath);
expect(sessionNew).not.toHaveBeenCalled();
});
test("rejects an SSH-only Catalog binding before persisting or starting a session", async () => {
const sessionNew = vi.fn(async () => ({ id: "must-not-exist" }));
const workspaceInputFingerprint = vi.fn(async () => "sha256:unused");
const ensure = vi.fn(async () => ({ ok: true }));
const createFor = vi.fn();
const abort = vi.fn(async () => {});
const markPersisted = vi.fn(async () => {});
const catalogRepository = new MemoryCatalogRepository();
await catalogRepository.create({
workspaceId: "ssh-workspace",
engine: "postgres",
databaseName: "postgres",
schema: "public",
binding: {
transport: "ssh_tunnel",
username: "reader",
sshHost: "bastion.internal",
sshPort: 22,
sshUsername: "tunnel",
sshTargetHost: "postgres.internal",
sshTargetPort: 5432,
},
});
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: { sessionNew, searchPack: async () => {} } as any,
thtRunner: { sessionNew, searchPack: async () => {}, workspaceInputFingerprint } as any,
catalogRepository,
readiness: { ensure } as any,
mgr: { get: () => undefined, createFor } as any,
getSettings: () => ({ workspace: "ssh-workspace" }) as any,
workspaceRuntimeSupport: vi.fn(() => false),
workspaceRuntimeSupport: vi.fn(() => true),
workspaceRegistry: {
acquireSessionRevision: vi.fn(async () => ({
workspace: {
workspace: { schema_version: 4, id: "ssh-workspace", name: "SSH", language: "en" },
dwh: {
engine: "postgres", database: "postgres", schema: "public",
supported_transports: ["ssh_tunnel"],
},
},
workspace: operationalWorkspace("ssh-workspace"),
revision: {
id: "ssh-workspace", commit: "a".repeat(40), blob: "b".repeat(40),
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/ssh-workspace.yaml`,
@@ -610,6 +686,7 @@ test("rejects an SSH-only workspace before persisting or starting a session", as
expect(response.statusCode).toBe(409);
expect(response.json()).toMatchObject({ code: "workspace_not_activatable" });
expect(ensure).not.toHaveBeenCalled();
expect(workspaceInputFingerprint).not.toHaveBeenCalled();
expect(sessionNew).not.toHaveBeenCalled();
expect(createFor).not.toHaveBeenCalled();
expect(abort).toHaveBeenCalledOnce();
+2 -1
View File
@@ -500,8 +500,9 @@ async function createRealRouteFixture() {
await git(author, ["init", "--initial-branch=main"]);
await git(author, ["config", "user.name", "Workspace Route Test"]);
await git(author, ["config", "user.email", "workspace-route@example.invalid"]);
const { dwh: _runtimeDwh, diagnostics: _runtimeDiagnostics, ...authoredWorkspace } = workspace;
const descriptor: CanonicalWorkspace = {
...workspace,
...authoredWorkspace,
evidence: {
source: {
type: "filesystem",
+5 -2
View File
@@ -52,8 +52,11 @@ test("Qdrant readiness uses only the internal URL and accepts the exact collecti
const request = vi.fn(async () => response(200, collection()));
await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ ok: true, state: "ready" });
expect(request).toHaveBeenCalledOnce();
expect(request.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd");
expect(request).toHaveBeenCalledTimes(2);
expect(request.mock.calls.map((call) => call[0])).toEqual([
"http://qdrant:6333/collections/psd-reference",
"http://qdrant:6333/collections/psd-memory",
]);
expect(request.mock.calls[0][1]).toMatchObject({ method: "GET", signal: expect.any(AbortSignal) });
});
+18 -30
View File
@@ -30,11 +30,11 @@ function ok(operation: string): WorkspaceOperationResult {
};
}
test("entrypoint emits exactly one pristine JSON document and maps success/block/failure exits", async () => {
test("entrypoint emits exactly one pristine JSON document and maps success/failure exits", async () => {
const service = {
inspect: vi.fn(async () => ok("inspect")),
preprocessDwh: vi.fn(async () => ({ ...ok("preprocess dwh"), status: "blocked", code: "manual_review_required" as const })),
run: vi.fn(async () => ({ ...ok("preprocess run"), status: "failed", code: "semantic_index_incompatible" as const })),
clear: vi.fn(async () => ok("preprocess clear")),
} as any;
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
@@ -42,13 +42,13 @@ test("entrypoint emits exactly one pristine JSON document and maps success/block
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "inspect", code: "ok" });
expect(inspectIo.stderr.join("")).toBe("");
const blockedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-dwh"], service, blockedIo)).toBe(3);
expect(JSON.parse(blockedIo.stdout.join(""))).toMatchObject({ code: "manual_review_required" });
const failedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-run"], service, failedIo)).toBe(1);
expect(JSON.parse(failedIo.stdout.join(""))).toMatchObject({ code: "semantic_index_incompatible" });
const clearIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-clear"], service, clearIo)).toBe(0);
expect(JSON.parse(clearIo.stdout.join(""))).toMatchObject({ operation: "preprocess clear", code: "ok" });
});
test("malformed stdin, unknown commands, and extra fields fail with exit 2 but still return bounded JSON", async () => {
@@ -84,31 +84,19 @@ test("raw exception text is redacted from stderr and stdout remains within the p
expect(captured.stderr.join("")).not.toContain("SELECT *");
});
test("vector-inspect and vector-rebuild dispatch to the service with the exact envelope", async () => {
test("retired partial preprocessing commands are rejected without dispatch", async () => {
const service = {
vectorInspect: vi.fn(async () => ok("vector inspect")),
vectorRebuild: vi.fn(async () => ok("vector rebuild")),
preprocessDwh: vi.fn(),
vectorInspect: vi.fn(),
vectorRebuild: vi.fn(),
} as any;
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-inspect"], service, inspectIo)).toBe(0);
expect(service.vectorInspect).toHaveBeenCalledWith({ workspaceId: "psd-clinical" });
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "vector inspect", code: "ok" });
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(0);
expect(service.vectorRebuild).toHaveBeenCalledWith({
workspaceId: "psd-clinical",
collection: "psd-clinical",
confirm: "psd-clinical",
destroy: true,
});
});
test("vector-rebuild without exact confirmation is refused by the service", async () => {
const service = {
vectorRebuild: vi.fn(async () => ({ ...ok("vector rebuild"), status: "failed", code: "semantic_index_incompatible" as const })),
} as any;
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "other", confirm: "other", destroy: true }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(1);
for (const command of ["preprocess-dwh", "vector-inspect", "vector-rebuild"]) {
const captured = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", command], service, captured)).toBe(2);
expect(JSON.parse(captured.stdout.join(""))).toMatchObject({ status: "failed", operation: command });
}
expect(service.preprocessDwh).not.toHaveBeenCalled();
expect(service.vectorInspect).not.toHaveBeenCalled();
expect(service.vectorRebuild).not.toHaveBeenCalled();
});
@@ -0,0 +1,259 @@
import Fastify from "fastify";
import { expect, test, vi } from "vitest";
import type { PrincipalContext } from "../src/auth/principal.js";
import type { CatalogRepository, WorkspaceDatabase } from "../src/catalog/types.js";
import {
readWorkspacePreprocessingStatus,
workspacePreprocessingRoutes,
type WorkspacePreprocessingRouteDeps,
} from "../src/routes/workspace-preprocessing.js";
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
const admin: PrincipalContext = {
issuer: "test",
subject: "admin",
roles: ["admin"],
permissions: ["session.use", "database.manage"],
isAdmin: true,
};
function database(overrides: Partial<WorkspaceDatabase> = {}): WorkspaceDatabase {
return {
id: "49b6491a-78bb-4cee-b27b-0efaf4419774",
workspaceId: "catalog-workspace",
engine: "postgres",
databaseName: "warehouse",
schema: "analytics",
version: 4,
createdAt: "2026-09-05T10:00:00.000Z",
updatedAt: "2026-09-05T10:00:00.000Z",
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
connectionStatus: "reachable",
schemaSyncedVersion: 4,
metadataContentRevision: 18,
preprocessingStatus: "failed",
...overrides,
};
}
function routeDeps(
current: () => WorkspaceDatabase | undefined,
overrides: Partial<WorkspacePreprocessingRouteDeps> = {},
): WorkspacePreprocessingRouteDeps {
const repository = {
getByWorkspace: vi.fn(async () => current()),
listSyncRuns: vi.fn(async () => []),
getActiveDescriptionGenerationRun: vi.fn(async () => undefined),
listSensitivityAnalysisRuns: vi.fn(async () => []),
} as unknown as CatalogRepository;
const registry = {
read: vi.fn(async () => ({
workspace: {
workspace: {
schema_version: 4,
id: "catalog-workspace",
name: "Catalog workspace",
language: "en",
},
},
revision: {
id: "catalog-workspace",
commit: "a".repeat(40),
blob: "b".repeat(40),
snapshotPath: "/data/workspaces/catalog-workspace.yaml",
},
})),
} as unknown as WorkspaceRegistry;
return {
repository,
registry,
service: {
run: vi.fn(async () => ({ status: "succeeded" })),
clear: vi.fn(async () => ({ status: "succeeded" })),
} as never,
inputFingerprint: {
workspaceInputFingerprint: vi.fn(async () => "sha256:current"),
} as never,
...overrides,
};
}
test("reports ready only when the Catalog revision and runtime fingerprint are current", async () => {
const ready = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
});
await expect(readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => ready),
)).resolves.toMatchObject({
state: "ready",
actionable: true,
clearable: true,
detail: "Catalog revision 18 is indexed.",
});
ready.preprocessingInputFingerprint = "sha256:old";
await expect(readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => ready),
)).resolves.toMatchObject({
state: "required",
actionable: true,
clearable: true,
detail: "Catalog revision 18 is not indexed.",
});
});
test("explains a current blocked prerequisite without inventing a run log", async () => {
const status = await readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => database({ schemaSyncedVersion: 3 })),
);
expect(status).toMatchObject({
state: "blocked",
actionable: false,
clearable: true,
detail: "Catalog synchronization is required.",
reason: "Database configuration v4 is newer than the latest Catalog synchronization v3.",
nextStep: "Open Database management and run Synchronize schema.",
});
expect(status).not.toHaveProperty("lastFailure");
});
test("exposes only the latest sanitized failed-run diagnostic", async () => {
const status = await readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(() => database({
preprocessingStatus: "failed",
preprocessingErrorCode: "schema_index_failed",
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
})),
);
expect(status).toMatchObject({
state: "failed",
actionable: true,
clearable: true,
reason: expect.stringContaining("schema indexing worker"),
lastFailure: {
stage: "schema_index",
errorCode: "schema_index_failed",
finishedAt: "2026-09-05T10:04:00.000Z",
},
});
expect(status).not.toHaveProperty("history");
});
test("reports the durable phase of the active preprocessing run", async () => {
const status = await readWorkspacePreprocessingStatus(
"catalog-workspace",
routeDeps(
() => database({ preprocessingStatus: "running" }),
{
readLatestJob: () => ({
status: "active",
completedStages: ["catalog_snapshot"],
}) as never,
},
),
);
expect(status).toMatchObject({
state: "running",
detail: "Building schema vectors and LSH indexes.",
progress: { stage: "schema_index", step: 2, totalSteps: 4 },
});
});
test("explicitly reruns preprocessing when the current Catalog input is already ready", async () => {
const ready = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
});
const deps = routeDeps(() => ready);
const app = Fastify();
app.addHook("preHandler", async (request) => { request.principal = admin; });
workspacePreprocessingRoutes(app, deps);
const response = await app.inject({
method: "POST",
url: "/workspaces/catalog-workspace/preprocessing",
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
expect(deps.service.run).toHaveBeenCalledTimes(1);
await app.close();
});
test("runs preprocessing once from the sanctioned admin endpoint and returns the new state", async () => {
let current = database();
const deps = routeDeps(() => current, {
service: {
run: vi.fn(async () => {
current = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
});
return { status: "succeeded" } as never;
}),
clear: vi.fn(async () => ({ status: "succeeded" } as never)),
},
});
const app = Fastify();
app.addHook("preHandler", async (request) => { request.principal = admin; });
workspacePreprocessingRoutes(app, deps);
const response = await app.inject({
method: "POST",
url: "/workspaces/catalog-workspace/preprocessing",
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
expect(deps.service.run).toHaveBeenCalledTimes(1);
await app.close();
});
test("clears only derived preprocessing data from the sanctioned admin endpoint", async () => {
let current = database({
preprocessingStatus: "succeeded",
preprocessedMetadataRevision: 18,
preprocessingInputFingerprint: "sha256:current",
});
const deps = routeDeps(() => current, {
service: {
run: vi.fn(),
clear: vi.fn(async () => {
current = database({
preprocessingStatus: "failed",
preprocessingErrorCode: "derived_data_cleared",
preprocessingFinishedAt: "2026-09-05T10:05:00.000Z",
});
return { status: "succeeded" } as never;
}),
},
});
const app = Fastify();
app.addHook("preHandler", async (request) => { request.principal = admin; });
workspacePreprocessingRoutes(app, deps);
const response = await app.inject({
method: "DELETE",
url: "/workspaces/catalog-workspace/preprocessing",
});
expect(response.statusCode).toBe(200);
expect(response.json()).toMatchObject({
state: "required",
clearable: false,
detail: "Reference vectors and LSH are empty. Memory is preserved.",
});
expect(deps.service.clear).toHaveBeenCalledTimes(1);
await app.close();
});
@@ -2,9 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test, vi } from "vitest";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { PreprocessingStateStore } from "../src/workspaces/preprocessing-state.js";
import { syncAnnotations } from "../src/workspaces/annotations-sync.js";
import { validateWorkspaceDescriptor } from "../src/workspaces/schema.js";
import {
WorkspacePreprocessingService,
type ChildProcessRequest,
@@ -16,73 +14,25 @@ afterEach(() => {
roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
});
const semanticRuntime = {
internalQdrantUrl: "http://qdrant:6333",
internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
internalEmbeddingModel: "qwen3-embedding:0.6b",
internalEmbeddingDimensions: 1024,
};
const workspace = validateWorkspaceDescriptor({
workspace: {
schema_version: 4,
id: "catalog-workspace",
name: "Catalog only",
language: "en",
},
});
const baseWorkspace = parseWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Runtime Lease
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
`);
const filesystemWorkspace = parseWorkspaceYaml(`${baseWorkspace ? '' : ''}workspace:
schema_version: 4
id: fs-workspace
name: Filesystem
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: filesystem
uri: fs-workspace/evidence
`);
const privateHttpWorkspace = parseWorkspaceYaml(`workspace:
schema_version: 4
id: http-workspace
name: Http
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: http
uris: [http://127.0.0.1/private.md]
authentication: none
connect_timeout_ms: 1000
read_timeout_ms: 2000
max_bytes: 100
max_redirects: 0
allow_private_hosts: true
max_cache_bytes: 100
`);
function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id) {
function runtime() {
return {
workspace,
workspaceId,
workspaceId: "catalog-workspace",
workspaceRevision: "a".repeat(40),
descriptorBlob: "b".repeat(40),
catalogBlob: "c".repeat(40),
configLease: {
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}-identitysuffix.yaml`,
workspaceId,
path: `/data/sessions/catalog-workspace/preprocessing/runtime-config/${"a".repeat(40)}.yaml`,
workspaceId: "catalog-workspace",
workspaceRevision: "a".repeat(40),
descriptorBlob: "b".repeat(40),
catalogBlob: "c".repeat(40),
@@ -90,21 +40,32 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
bindingDigest: "sha256:bindings",
semanticQdrantUrl: "http://qdrant:6333",
effectiveConfig: {
schemaVersion: 2,
schemaVersion: 3,
dwh: {
engine: "postgres",
database: "analytics",
schema: "mart",
database: "warehouse",
schema: "analytics",
transport: "postgres_direct",
host: "dwh.internal",
host: "db.internal",
port: 5432,
user: "reader",
},
vector: { collection: workspaceId, dimensions: 1024, distance: "cosine" },
embedding: { id: "ollama/qwen3-embedding:0.6b", model: "qwen3-embedding:0.6b", dimensions: 1024 },
vector: {
collections: {
reference: "catalog-workspace-reference",
memory: "catalog-workspace-memory",
},
dimensions: 1024,
distance: "cosine",
},
embedding: {
id: "ollama/qwen3-embedding:0.6b",
model: "qwen3-embedding:0.6b",
dimensions: 1024,
},
roots: { artifacts: "/data/artifacts", indexes: "/data/indexes" },
},
effectiveConfigIdentity: "workspace://psd-clinical@v1:" + "d".repeat(64),
effectiveConfigIdentity: "workspace://catalog-workspace@v1:" + "d".repeat(64),
configFingerprint: "sha256:" + "e".repeat(64),
inputFingerprint: "sha256:" + "f".repeat(64),
release: () => undefined,
@@ -112,385 +73,208 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
};
}
function fixture(workspace = baseWorkspace) {
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
roots.push(dataRoot);
const requests: ChildProcessRequest[] = [];
const runChild = vi.fn(async (request: ChildProcessRequest) => {
requests.push(request);
return { exitCode: 0, stdout: JSON.stringify({ status: "succeeded" }), stderr: "" };
});
const service = new WorkspacePreprocessingService({
dataRoot,
acquireActiveRuntime: async () => runtime(workspace),
runChild,
listSessions: async () => [],
semanticPreflight: async () => ({ ok: true }),
evidencePreflight: async () => ({ ok: true }),
});
return { dataRoot, runChild, requests, service };
const database = {
id: "database-1",
workspaceId: "catalog-workspace",
engine: "postgres",
databaseName: "warehouse",
schema: "analytics",
version: 4,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
connectionStatus: "reachable",
schemaSyncedVersion: 4,
metadataContentRevision: 9,
preprocessingStatus: "running",
} as const;
function repository(overrides: Record<string, unknown> = {}) {
const finishPreprocessing = vi.fn(async () => ({
...database,
preprocessingStatus: "succeeded" as const,
preprocessedMetadataRevision: 9,
}));
return {
beginPreprocessing: vi.fn(async () => ({ kind: "started" as const, database })),
finishPreprocessing,
clearPreprocessing: vi.fn(async () => ({ kind: "cleared" as const, database })),
getByWorkspace: vi.fn(async () => database),
listTables: vi.fn(async () => [{
id: "table-1", databaseId: database.id, name: "orders",
description: "Curated orders", generatedDescription: "Generated orders",
sourceComment: "PostgreSQL orders", version: 1,
createdAt: database.createdAt, updatedAt: database.updatedAt,
lastSyncedDatabaseVersion: 4, lastSyncedAt: database.updatedAt,
}]),
listColumns: vi.fn(async () => [{
id: "column-1", tableId: "table-1", name: "customer_id", ordinalPosition: 1,
dataType: "uuid", isNullable: false, defaultExpression: null,
primaryKeyPosition: null, isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1,
sourceComment: "PostgreSQL customer", description: null,
generatedDescription: "Generated customer", sensitive: true,
sensitivityReason: "identifier", lastSyncedDatabaseVersion: 4,
lastSyncedAt: database.updatedAt, version: 1,
createdAt: database.createdAt, updatedAt: database.updatedAt,
}]),
listRelationships: vi.fn(async () => []),
listLogicalRelationships: vi.fn(async () => []),
...overrides,
} as any;
}
test("preprocess dwh uses fixed argv and resumes outer state without rerunning a completed stage", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
stderr: "",
});
const first = await f.service.preprocessDwh({ workspaceId: "psd-clinical" });
expect(first).toMatchObject({
status: "succeeded",
code: "ok",
operation: "preprocess dwh",
completedStages: ["dwh"],
childRuns: { dwh: "d".repeat(32) },
});
expect((f.runChild.mock.calls[0]![0] as ChildProcessRequest).argv).toEqual([
"preprocess", "dwh", "--steps", "introspect,lsh", "--json", "-c", "/dev/fd/3",
]);
const second = await f.service.preprocessDwh({
workspaceId: "psd-clinical",
resumeRunId: first.runId!,
});
expect(second.status).toBe("unchanged");
expect(f.runChild).toHaveBeenCalledTimes(1);
});
test("schema suggest-fks publishes a candidate artifact and blocks full runs for manual review", async () => {
const f = fixture();
f.runChild
.mockResolvedValueOnce({
function service(options: {
repository?: any;
runChild?: (request: ChildProcessRequest) => Promise<{
exitCode: number; stdout: string; stderr: string;
}>;
semanticPreflight?: () => Promise<
{ ok: true } | { ok: false; code: "semantic_index_incompatible" }
>;
} = {}) {
const dataRoot = mkdtempSync(join(tmpdir(), "tht-catalog-preprocessing-"));
roots.push(dataRoot);
return new WorkspacePreprocessingService({
dataRoot,
acquireActiveRuntime: async () => runtime(),
runChild: options.runChild ?? (async () => ({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: []\n",
}),
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
stderr: "",
})),
semanticPreflight: options.semanticPreflight ?? (async () => ({ ok: true })),
evidencePreflight: async () => ({ ok: true }),
catalogRepository: options.repository,
});
}
test("complete preprocessing snapshots Catalog metadata and commits its PostgreSQL state", async () => {
const catalog = repository();
const runChild = vi.fn(async (request: ChildProcessRequest) => {
const snapshotPath = request.argv[request.argv.indexOf("--catalog-metadata") + 1]!;
const snapshot = JSON.parse(readFileSync(snapshotPath, "utf8"));
expect(snapshot).toMatchObject({
workspaceId: "catalog-workspace",
databaseName: "warehouse",
metadataContentRevision: 9,
tables: [{
name: "orders",
description: "Curated orders",
descriptionSource: "curated",
columns: [{
name: "customer_id",
description: "Generated customer",
descriptionSource: "generated",
sensitive: true,
}],
}],
});
const result = await f.service.run({ workspaceId: "psd-clinical" });
expect(result).toMatchObject({
status: "blocked",
code: "manual_review_required",
completedStages: ["dwh", "fk_suggest"],
});
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv[0])).toEqual(["preprocess", "schema"]);
});
test("schema check requires the exact candidate digest and stages annotations via a temp file without recording a review", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: []\n",
}),
stderr: "",
});
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
await expect(f.service.checkSchema({
workspaceId: "psd-clinical",
annotationsYaml: "tables: {}\n",
reviewedCandidatesDigest: "sha256:" + "f".repeat(64),
})).resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
const reviewedDigest = suggest.artifactIdentities![0]!.digest;
let stagedPath = "";
f.runChild.mockImplementationOnce(async (request: ChildProcessRequest) => {
stagedPath = request.argv[request.argv.indexOf("--annotations") + 1]!;
expect(readFileSync(stagedPath, "utf8")).toBe("tables: {}\n");
expect(request.argv).toEqual([
"schema", "check", "--annotations", stagedPath,
"--reviewed-candidates", reviewedDigest,
"--json", "-c", "/dev/fd/3",
]);
return {
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
orphan_count: 0,
annotations_digest: "sha256:annotations",
reviewed_candidates_digest: reviewedDigest,
}),
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
stderr: "",
};
});
const checked = await f.service.checkSchema({
workspaceId: "psd-clinical",
annotationsYaml: "tables: {}\n",
reviewedCandidatesDigest: reviewedDigest,
});
expect(checked).toMatchObject({ status: "succeeded", code: "ok" });
expect(() => readFileSync(stagedPath, "utf8")).toThrow();
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
expect(state.readFkReview(suggest.runId!)).toBeUndefined();
});
test("index schema fails closed when semantic preflight refuses the collection", async () => {
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
roots.push(dataRoot);
const runChild = vi.fn();
const service = new WorkspacePreprocessingService({
dataRoot,
acquireActiveRuntime: async () => runtime(baseWorkspace),
runChild,
listSessions: async () => [],
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
evidencePreflight: async () => ({ ok: true }),
const result = await service({ repository: catalog, runChild }).run({
workspaceId: "catalog-workspace",
});
const result = await service.indexSchema({ workspaceId: "psd-clinical" });
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
expect(runChild).not.toHaveBeenCalled();
});
test("filesystem Evidence proceeds after materialization and private HTTP hosts outside the allowlist are refused", async () => {
const filesystem = fixture(filesystemWorkspace);
filesystem.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1 } }),
stderr: "",
});
const materialized = await filesystem.service.preprocessEvidence({ workspaceId: "fs-workspace" });
expect(materialized).toMatchObject({ status: "succeeded", code: "ok" });
expect(filesystem.runChild).toHaveBeenCalledTimes(1);
const httpDataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
roots.push(httpDataRoot);
const httpService = new WorkspacePreprocessingService({
dataRoot: httpDataRoot,
acquireActiveRuntime: async () => runtime(privateHttpWorkspace, "http-workspace"),
runChild: vi.fn(),
listSessions: async () => [],
semanticPreflight: async () => ({ ok: true }),
evidencePreflight: async () => ({ ok: true }),
httpPrivateHostAllowlist: ["metadata.internal"],
});
const refused = await httpService.preprocessEvidence({ workspaceId: "http-workspace" });
expect(refused).toMatchObject({ status: "failed", code: "egress_policy_refused" });
});
test("full runs follow the explicit order and finish unchanged when no Evidence source exists", async () => {
const f = fixture();
f.runChild
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 0,
candidate_digest: "sha256:" + "0".repeat(64),
candidate_yaml: "tables: []\n",
}),
stderr: "",
})
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 },
}),
stderr: "",
});
const result = await f.service.run({ workspaceId: "psd-clinical" });
expect(result).toMatchObject({
status: "succeeded",
code: "ok",
completedStages: ["dwh", "fk_suggest", "schema_index"],
warnings: ["workspace has no Evidence source"],
completedStages: ["catalog_snapshot", "catalog_metadata", "lsh", "schema_index"],
});
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv.slice(0, 2).join(" "))).toEqual([
"preprocess dwh",
"schema suggest-fks",
"vector index-schema",
]);
expect(runChild).toHaveBeenCalledWith(expect.objectContaining({
argv: [
"preprocess", "catalog", "--catalog-metadata",
expect.stringMatching(/\/catalog-metadata\.json$/),
"--json", "-c", "/dev/fd/3",
],
}));
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
"catalog-workspace",
9,
"sha256:" + "f".repeat(64),
{ status: "succeeded" },
);
});
test("schema accept validates the synced Git blob and records the review", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: {}\n",
}),
stderr: "",
});
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
const runId = suggest.runId!;
const candidateDigest = suggest.artifactIdentities![0]!.digest;
const synced = syncAnnotations({
dataRoot: f.dataRoot,
workspaceId: "psd-clinical",
commit: "a".repeat(40),
blobId: "b".repeat(40),
contents: Buffer.from("tables: {}\n"),
});
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
orphan_count: 0,
annotations_digest: synced.contentDigest,
reviewed_candidates_digest: candidateDigest,
}),
stderr: "",
});
const result = await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
expect(result).toMatchObject({ status: "succeeded", code: "ok", operation: "schema accept" });
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
expect(state.readFkReview(runId)).toMatchObject({
reviewedCandidatesDigest: candidateDigest,
annotationsDigest: synced.contentDigest,
workspaceRevision: "a".repeat(40),
blobId: "b".repeat(40),
});
test("preprocessing fails closed when the PostgreSQL Catalog is unavailable", async () => {
const result = await service().run({ workspaceId: "catalog-workspace" });
expect(result).toMatchObject({ status: "failed", code: "catalog_not_ready" });
});
test("schema accept fails closed without --yes, for an unknown run, or with no synced annotations", async () => {
const f = fixture();
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({
status: "succeeded",
candidate_count: 1,
candidate_digest: "sha256:" + "e".repeat(64),
candidate_yaml: "tables: {}\n",
}),
stderr: "",
test("preprocessing refuses a concurrent Catalog run without touching derived data", async () => {
const catalog = repository({
beginPreprocessing: vi.fn(async () => ({ kind: "already_running" as const })),
});
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
const runId = suggest.runId!;
const runChild = vi.fn();
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: false }))
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
const result = await service({ repository: catalog, runChild }).run({
workspaceId: "catalog-workspace",
});
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId: "e".repeat(32), yes: true }))
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true }))
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
expect(f.runChild).toHaveBeenCalledTimes(1);
expect(result).toMatchObject({ status: "failed", code: "preprocessing_conflict" });
expect(runChild).not.toHaveBeenCalled();
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
});
test("full runs continue after schema accept only when the accepted blob matches the current revision", async () => {
const f = fixture();
const revision = "a".repeat(40);
f.runChild
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
stderr: "",
});
const blocked = await f.service.run({ workspaceId: "psd-clinical" });
expect(blocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
const runId = blocked.runId!;
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
const candidateDigest = state.readFkCandidates(runId)!.digest;
const synced = syncAnnotations({
dataRoot: f.dataRoot,
workspaceId: "psd-clinical",
commit: revision,
blobId: "b".repeat(40),
contents: Buffer.from("tables: {}\n"),
test("preprocessing records a failed PostgreSQL state when its hidden worker fails", async () => {
const catalog = repository();
const instance = service({
repository: catalog,
runChild: async () => ({ exitCode: 1, stdout: "", stderr: "private failure" }),
});
// Accept writes the review; the resume then passes the gate and reaches schema indexing.
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", orphan_count: 0, annotations_digest: synced.contentDigest, reviewed_candidates_digest: candidateDigest }),
stderr: "",
});
await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
f.runChild.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 } }),
stderr: "",
});
const resumed = await f.service.run({ workspaceId: "psd-clinical", resumeRunId: runId });
expect(resumed).toMatchObject({ status: "succeeded", code: "ok" });
// A review whose accepted blob digest no longer matches the current revision stays blocked.
const second = fixture();
second.runChild
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
.mockResolvedValueOnce({
exitCode: 0,
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
stderr: "",
});
const secondBlocked = await second.service.run({ workspaceId: "psd-clinical" });
const secondRunId = secondBlocked.runId!;
const secondState = new PreprocessingStateStore({ dataRoot: second.dataRoot, workspaceId: "psd-clinical" });
const secondCandidate = secondState.readFkCandidates(secondRunId)!.digest;
secondState.writeFkReview(secondRunId, {
reviewedCandidatesDigest: secondCandidate,
annotationsDigest: "sha256:" + "0".repeat(64),
workspaceRevision: revision,
blobId: "b".repeat(40),
});
const stillBlocked = await second.service.run({ workspaceId: "psd-clinical", resumeRunId: secondRunId });
expect(stillBlocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
await expect(instance.run({ workspaceId: "catalog-workspace" })).rejects.toThrow(
"workspace child failed",
);
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
"catalog-workspace",
9,
"sha256:" + "f".repeat(64),
{ status: "failed", errorCode: "schema_index_failed" },
);
});
test("vector rebuild recreates the full collection contract including keyword indexes", async () => {
const f = fixture();
// mock fetch: DELETE ok, then reconcileCollection self-heals create + indexes (real fetch in deps)
const calls: string[] = [];
const fakeFetch = async (url: string, init?: any) => {
calls.push(`${init?.method ?? "GET"} ${url}`);
if ((init?.method ?? "GET") === "DELETE") return new Response("", { status: 200 });
if (url.endsWith("/collections/psd-clinical") && init?.method === "PUT") return new Response("", { status: 200 });
if (url.endsWith("/collections/psd-clinical") && init?.method === "GET") {
return new Response(JSON.stringify({ result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, payload_schema: { content_hash: { data_type: "keyword" }, document_id: { data_type: "keyword" }, kind: { data_type: "keyword" }, record_key: { data_type: "keyword" }, record_kind: { data_type: "keyword" }, vector_generation: { data_type: "keyword" }, workspace_id: { data_type: "keyword" }, workspace_revision: { data_type: "keyword" } } } }), { status: 200 });
}
if (url.endsWith("/collections/psd-clinical/index") && init?.method === "PUT") return new Response("", { status: 200 });
return new Response(JSON.stringify({ result: {} }), { status: 200 });
};
const service = new WorkspacePreprocessingService({
dataRoot: f.dataRoot,
acquireActiveRuntime: async () => runtime(baseWorkspace),
runChild: vi.fn(),
listSessions: async () => [],
semanticPreflight: async () => ({ ok: true }),
evidencePreflight: async () => ({ ok: true }),
test("semantic preflight failure is persisted before returning", async () => {
const catalog = repository();
const result = await service({
repository: catalog,
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
}).run({ workspaceId: "catalog-workspace" });
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
"catalog-workspace",
9,
"sha256:" + "f".repeat(64),
{ status: "failed", errorCode: "semantic_index_incompatible" },
);
});
test("clear invalidates Catalog readiness before clearing only derived worker data", async () => {
const catalog = repository();
const runChild = vi.fn(async () => ({
exitCode: 0,
stdout: JSON.stringify({ counts: { referenceCollections: 1, derivedPaths: 3 } }),
stderr: "",
}));
const result = await service({ repository: catalog, runChild }).clear({
workspaceId: "catalog-workspace",
});
// replace global fetch used by vectorRebuild/reconcileCollection
const original = globalThis.fetch;
globalThis.fetch = fakeFetch as any;
try {
const result = await service.vectorRebuild({ workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true });
expect(result).toMatchObject({ status: "succeeded", code: "ok" });
} finally {
globalThis.fetch = original;
}
expect(calls.some((c) => c.startsWith("DELETE "))).toBe(true);
expect(catalog.clearPreprocessing).toHaveBeenCalledWith("catalog-workspace");
expect(runChild).toHaveBeenCalledWith({
argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
configPath: expect.any(String),
});
expect(result).toMatchObject({
status: "succeeded",
code: "ok",
operation: "preprocess clear",
counts: { referenceCollections: 1, derivedPaths: 3 },
});
});
@@ -47,6 +47,15 @@ test("job state creates durable 0600 JSON and enforces same-revision resume", as
embeddingId: "ollama/qwen3-embedding:0.6b",
embeddingDimensions: 1024,
});
expect(store.readLatestJob()).toMatchObject({
runId: job.runId,
status: "active",
completedStages: [],
});
job.completedStages.push("catalog_snapshot");
store.writeJob(job);
expect(store.readLatestJob()?.completedStages).toEqual(["catalog_snapshot"]);
await expect(store.beginJob({
operation: "preprocess dwh",
+2 -131
View File
@@ -19,11 +19,6 @@ const validYaml = `workspace:
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
`;
function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
@@ -34,119 +29,12 @@ function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
`);
}
function withDwhRestTransport(source: string): string {
return source.replace(
"supported_transports: [postgres_direct]",
"supported_transports: [postgres_direct, rest_api]",
);
}
function withDwhRestDiagnostic(source: string): string {
return withDwhRestTransport(source).concat(`diagnostics:
dwh_rest:
method: GET
path: /health
auth: none
response:
database: database
schema: schema
`);
}
function withEmbeddingDiagnostic(source: string): string {
return source.concat(`diagnostics:
embedding:
method: GET
path: /models
auth: none
response:
model: model
dimensions: dimensions
`);
}
function withDwhRestAndEmbeddingDiagnostics(source: string): string {
return withDwhRestTransport(source).concat(`diagnostics:
dwh_rest:
method: GET
path: /health
auth: none
response:
database: database
schema: schema
embedding:
method: GET
path: /models
auth: none
response:
model: model
dimensions: dimensions
`);
}
function withVectorRestTransport(source: string): string {
return source.replace(
"supported_transports: [pgvector_direct]",
"supported_transports: [pgvector_direct, rest_api]",
);
}
function withVectorMetadataDiagnostic(source: string): string {
return withVectorRestTransport(source).concat(`diagnostics:
vector_rest:
metadata:
method: GET
path: /metadata
auth: none
response:
collection: collection
dimensions: dimensions
distance: distance
`);
}
function withReversibleVectorProbe(source: string): string {
return withVectorRestTransport(source).concat(`diagnostics:
vector_rest:
metadata:
method: GET
path: /metadata
auth: none
response:
collection: collection
dimensions: dimensions
distance: distance
reversible_probe:
method: POST
path: /probe
auth: bearer
response:
operation: operation
`);
}
function legacyV1Yaml(source = validYaml): string {
return source
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
.replace(" dimensions: 1024", " dimensions: 768")
.replace(" provider: ollama_internal", " provider: ollama_compatible")
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
.replace(" dimensions: 1024", " dimensions: 768")
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
.replace("schema_version: 4", "schema_version: 1");
return source.replace("schema_version: 4", "schema_version: 1");
}
function legacyV2Yaml(source = validYaml): string {
return source
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
.replace(" dimensions: 1024", " dimensions: 768")
.replace(" provider: ollama_internal", " provider: ollama_compatible")
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
.replace(" dimensions: 1024", " dimensions: 768")
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
.replace("schema_version: 4", "schema_version: 2");
return source.replace("schema_version: 4", "schema_version: 2");
}
const runFile = promisify(execFile);
@@ -605,23 +493,6 @@ test("keeps content-only historical descriptor revisions distinguishable by comm
expect(oldPinned.workspace).toEqual(newPinned.workspace);
});
test.each([
["adds", validYaml, withDwhRestDiagnostic(validYaml)],
["removes", withDwhRestDiagnostic(validYaml), validYaml],
])("pulls a curator change that %s a diagnostics branch without API rewrite", async (_operation, baseSource, remoteSource) => {
const remote = await fixture(baseSource);
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
await registry.bootstrap();
const initial = await registry.read("psd-clinical");
writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), remoteSource);
await git(remote.source, ["add", "psd-clinical/workspace.yaml"]);
await git(remote.source, ["commit", "-m", `Registry ${_operation} diagnostic branch`]);
await git(remote.source, ["push", "origin", "main"]);
await registry.pull();
const updated = await registry.read("psd-clinical");
expect(updated.revision.commit).not.toBe(initial.revision.commit);
});
test.each([
["v1", legacyV1Yaml()],
["v2", legacyV2Yaml()],
@@ -1,6 +1,5 @@
import { execFile } from "node:child_process";
import {
chmodSync,
existsSync,
mkdtempSync,
mkdirSync,
@@ -27,6 +26,7 @@ import {
renderActiveWorkspaceRuntime,
renderWorkspaceRuntimeFromSnapshotPath,
} from "../src/workspaces/runtime-config-lease.js";
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
const runFile = promisify(execFile);
const roots: string[] = [];
@@ -70,11 +70,6 @@ workspaces: [{id: psd-clinical, name: Runtime Lease}]
id: psd-clinical
name: Runtime Lease
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
evidence:
source:
type: filesystem
@@ -88,9 +83,6 @@ evidence:
await git(source, ["push", "origin", "main"]);
mkdirSync(secretRoot);
const passwordFile = join(secretRoot, "dwh-password");
writeFileSync(passwordFile, "secret", { mode: 0o600 });
chmodSync(passwordFile, 0o600);
mkdirSync(dataRoot);
const registryConfig: WorkspaceRegistryConfig = {
@@ -107,12 +99,31 @@ evidence:
const registry = new WorkspaceRegistry(registryConfig);
await registry.bootstrap();
const revision = (await registry.list())[0];
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse.internal");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "reader");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE", passwordFile);
const workspaceSecretStore = new WorkspaceSecretStore({
root: join(root, "vault"),
runtimeRoot: join(root, "runtime-secrets"),
installationId: "test",
});
workspaceSecretStore.putMany("psd-clinical", { "catalog.dwh.password": "secret" });
const catalogDatabase = {
id: "database-1",
workspaceId: "psd-clinical",
engine: "postgres" as const,
databaseName: "analytics",
schema: "mart",
binding: {
transport: "postgres_direct" as const,
host: "warehouse.internal",
port: 5432,
username: "reader",
},
version: 1,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
connectionStatus: "reachable" as const,
metadataContentRevision: 1,
preprocessingStatus: "failed" as const,
};
return {
dataRoot,
@@ -121,6 +132,8 @@ evidence:
registry,
registryConfig,
revision,
workspaceSecretStore,
catalogDatabase,
};
}
@@ -140,6 +153,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const active = await renderActiveWorkspaceRuntime({
workspaceId: "psd-clinical",
@@ -150,6 +165,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
expect(active.renderedConfig).toBe(direct.renderedConfig);
@@ -158,27 +175,6 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/);
});
test("renders a revision-qualified annotations root for the active revision", async () => {
const f = await fixture();
const active = await renderActiveWorkspaceRuntime({
workspaceId: "psd-clinical",
registry: f.registry,
registryConfig: f.registryConfig,
harnessDir: f.harnessDir,
configPath: "config/tht.yaml",
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
});
const rendered = parse(active.renderedConfig) as Record<string, any>;
expect(rendered.paths.annotations_root).toBe(
join(f.dataRoot, "sessions", "psd-clinical", "revisions", f.revision.commit, "artifacts"),
);
expect(rendered.roots.annotations_root).toBe(rendered.paths.annotations_root);
expect(rendered.paths.artifacts).toBe(join(f.dataRoot, "sessions", "psd-clinical", "artifacts"));
});
test("deterministic operator leases are keyed by logical identity and stable across calls", async () => {
const f = await fixture();
const first = await publishDeterministicRuntimeConfigLease({
@@ -190,6 +186,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const second = await publishDeterministicRuntimeConfigLease({
workspaceId: "psd-clinical",
@@ -200,6 +198,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const suffix = first.inputFingerprint.slice(7, 23);
@@ -236,7 +236,11 @@ test("deterministic operator leases are keyed by logical identity and stable acr
path: first.path,
});
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal");
const changedDatabase = {
...f.catalogDatabase,
binding: { ...f.catalogDatabase.binding, host: "warehouse-two.internal" },
version: 2,
};
const changed = await publishDeterministicRuntimeConfigLease({
workspaceId: "psd-clinical",
registry: f.registry,
@@ -246,6 +250,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: changedDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
expect(changed.path).not.toBe(first.path);
expect(changed.inputFingerprint).not.toBe(first.inputFingerprint);
@@ -267,6 +273,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
})).toThrow(/trusted runtime snapshot/i);
expect(() => renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: symlink,
@@ -275,6 +283,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
})).toThrow(/trusted runtime snapshot/i);
});
@@ -288,6 +298,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const lease = await publishDeterministicRuntimeConfigLease({
workspaceId: "psd-clinical",
@@ -298,6 +310,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const sessionCanonical = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(parse(session.renderedConfig)));
@@ -319,6 +333,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
const firstIdentity = firstLease.effectiveConfigIdentity;
@@ -341,6 +357,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
dataRoot: f.dataRoot,
secretRoots: f.registryConfig.secretRoots,
semanticRuntime,
catalogDatabase: f.catalogDatabase,
workspaceSecretStore: f.workspaceSecretStore,
});
expect(secondLease.workspaceRevision).toBe(current.commit);
+53 -39
View File
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import {
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync,
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
@@ -25,11 +25,6 @@ const canonicalWorkspace = `workspace:
id: psd-clinical
name: Runtime handoff
language: en
dwh:
engine: postgres
database: analytics
schema: mart
supported_transports: [postgres_direct]
`;
const filesystemWorkspace = `${canonicalWorkspace}evidence:
@@ -106,6 +101,38 @@ async function fixture(workspaceSource = filesystemWorkspace) {
const registry = new WorkspaceRegistry(registryConfig);
await registry.bootstrap();
const revision = (await registry.list())[0];
const workspaceSecretStore = new WorkspaceSecretStore({
root: join(root, "workspace-secrets"),
runtimeRoot: join(root, "workspace-secret-runtime"),
installationId: "test",
});
workspaceSecretStore.putMany("psd-clinical", {
"catalog.dwh.password": "dwh-password-value",
"evidence.signed_urls": secretContents["evidence-signed-urls.json"],
"evidence.access_key": secretContents["evidence-access"],
"evidence.secret_key": secretContents["evidence-secret"],
"evidence.session_token": secretContents["evidence-token"],
});
const catalogDatabase = {
id: "database-1",
workspaceId: "psd-clinical",
engine: "postgres" as const,
databaseName: "analytics",
schema: "mart",
binding: {
transport: "postgres_direct" as const,
host: "dwh.invalid",
port: 5432,
username: "reader",
},
version: 1,
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
connectionStatus: "reachable" as const,
metadataContentRevision: 1,
preprocessingStatus: "failed" as const,
};
const catalogRepository = { getByWorkspace: async () => catalogDatabase } as any;
const environment = {
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.invalid",
@@ -119,7 +146,10 @@ async function fixture(workspaceSource = filesystemWorkspace) {
};
for (const [name, value] of Object.entries(environment)) vi.stubEnv(name, value);
vi.stubEnv("THT_HOME", join(root, "home"));
return { root, source, dataRoot, secretRoot, registry, registryConfig, revision };
return {
root, source, dataRoot, secretRoot, registry, registryConfig, revision,
workspaceSecretStore, catalogDatabase, catalogRepository,
};
}
function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
@@ -130,6 +160,8 @@ function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
dataRoot: f.dataRoot,
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
secretRoots: f.registryConfig.secretRoots,
workspaceSecretStore: f.workspaceSecretStore,
catalogRepository: f.catalogRepository,
} as any);
}
@@ -165,7 +197,7 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
runtimeRoot,
installationId: "test",
});
secretStore.put("psd-clinical", "dwh.password", "vault-runtime-password");
secretStore.put("psd-clinical", "catalog.dwh.password", "vault-runtime-password");
const runner = new ThtRunner({
thtBin,
harnessDir,
@@ -174,9 +206,10 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
secretRoots: f.registryConfig.secretRoots,
workspaceSecretStore: secretStore,
catalogRepository: f.catalogRepository,
} as any);
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const rendered = parse(readFileSync(lease.path, "utf8")) as {
database: { password_file: string };
};
@@ -185,30 +218,11 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
expect(existsSync(rendered.database.password_file)).toBe(false);
});
test("ThtRunner binds and cleans the effective relationship snapshot with its runtime lease", async () => {
const f = await fixture();
const runner = runnerFor(f);
const relationships = JSON.stringify({
schemaVersion: 1,
workspaceId: "psd-clinical",
relationships: [],
});
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath, relationships);
const rendered = parse(readFileSync(lease.path, "utf8")) as {
paths: { effective_relationships: string };
};
expect(readFileSync(rendered.paths.effective_relationships, "utf8")).toBe(relationships);
lease.release();
expect(existsSync(rendered.paths.effective_relationships)).toBe(false);
});
test("separate runtime leases hand off byte-identical revision Evidence configs accepted by tht", async () => {
const f = await fixture();
const runner = runnerFor(f);
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const second = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const second = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const expectedRoot = join(
f.registryConfig.root,
"snapshots",
@@ -263,7 +277,7 @@ test("separate runtime leases hand off byte-identical revision Evidence configs
test("real Evidence-content-only commit changes runtime identity and root with identical descriptor YAML", async () => {
const f = await fixture();
const runner = runnerFor(f);
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const descriptorBefore = readFileSync(f.revision.snapshotPath, "utf8");
writeFileSync(
join(f.source, "psd-clinical", "evidence", "guide.md"),
@@ -274,7 +288,7 @@ test("real Evidence-content-only commit changes runtime identity and root with i
await git(f.source, ["push", "origin", "main"]);
await f.registry.pull();
const current = (await f.registry.list())[0];
const second = runner.acquireWorkspaceRuntime(current.snapshotPath);
const second = await runner.acquireWorkspaceRuntime(current.snapshotPath);
try {
expect(current.commit).not.toBe(f.revision.commit);
@@ -317,13 +331,13 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
max_cache_bytes: 67890
`));
const runner = runnerFor(f);
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
try {
const yaml = readFileSync(lease.path, "utf8");
expect(parse(yaml).evidence.sources).toEqual([{
type: "http",
provenance_urls: ["https://evidence.example.test/guide.md"],
signed_urls_file: realpathSync(join(f.secretRoot, "evidence-signed-urls.json")),
signed_urls_file: expect.stringContaining("/workspace-secret-runtime/"),
connect_timeout: 1.25,
read_timeout: 30.001,
max_bytes: 12_345,
@@ -343,7 +357,7 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
}
});
test("static S3 Evidence resolves only configured secret-root file paths", async () => {
test("static S3 Evidence resolves only ephemeral vault materializations", async () => {
const f = await fixture(evidenceWorkspace(` type: s3
uri: s3://clinical-evidence/published/
endpoint_url: https://s3.example.test/
@@ -361,7 +375,7 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
retain_published_generations: 7
`));
const runner = runnerFor(f);
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
try {
const yaml = readFileSync(lease.path, "utf8");
expect(parse(yaml).evidence.sources).toEqual([{
@@ -370,9 +384,9 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
prefix: "published/",
endpoint_url: "https://s3.example.test/",
region: "eu-west-1",
access_key_file: realpathSync(join(f.secretRoot, "evidence-access")),
secret_key_file: realpathSync(join(f.secretRoot, "evidence-secret")),
session_token_file: realpathSync(join(f.secretRoot, "evidence-token")),
access_key_file: expect.stringContaining("/workspace-secret-runtime/"),
secret_key_file: expect.stringContaining("/workspace-secret-runtime/"),
session_token_file: expect.stringContaining("/workspace-secret-runtime/"),
trusted_endpoint: true,
allow_private_endpoint: true,
allow_insecure_endpoint: false,
@@ -10,9 +10,9 @@ import {
type SemanticRuntimeConfig,
} from "../src/workspaces/runtime-renderer.js";
import { supportsSessionRuntime } from "../src/workspaces/bindings.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const workspaceV4 = parseWorkspaceYaml(`workspace:
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -70,7 +70,14 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu
},
dwh: { type: "postgres_direct" },
resources: {
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
vector: {
engine: "qdrant",
base_url: "http://qdrant:6333",
collections: {
reference: "psd-clinical-reference",
memory: "psd-clinical-memory",
},
},
embeddings: {
provider: "ollama_internal", base_url: "http://embedding:11434",
id: "ollama/qwen3-embedding:0.6b",
@@ -133,7 +140,7 @@ function evidenceWorkspace(
policy?: Record<string, unknown>,
evidenceSchemaVersion?: number,
) {
return parseWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
return parseRuntimeWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
evidenceSchemaVersion === undefined ? "" : `\n schema_version: ${evidenceSchemaVersion}`
}\n source: ${JSON.stringify(source)}${
policy === undefined ? "" : `\n policy: ${JSON.stringify(policy)}`
@@ -8,12 +8,12 @@ import {
resolveRuntimeBindingsWithWorkspaceSecrets,
} from "../src/workspaces/secret-requirements.js";
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const roots: string[] = [];
function workspace(extra = "") {
return parseWorkspaceYaml(`workspace:
return parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
+2 -5
View File
@@ -23,14 +23,11 @@ test("strict workspace v4 rejects model-bearing v3 descriptors", () => {
expect(() => parseWorkspaceYaml(legacy)).toThrow();
});
test("v3 to v4 migration removes only model/vector policy and bumps the version", () => {
test("v3 to v4 migration removes database/model policy and bumps the version", () => {
const migrated = migrateWorkspaceV3Yaml(legacy);
const workspace = parseWorkspaceYaml(migrated);
expect(workspace.workspace).toMatchObject({ schema_version: 4, id: "abc" });
expect(migrated).not.toMatch(/semantic_index|llm_policy/);
expect(workspace.dwh).toEqual({
engine: "postgres", database: "warehouse", schema: "public",
supported_transports: ["postgres_direct"],
});
expect(workspace.dwh).toBeUndefined();
});
+5 -5
View File
@@ -7,9 +7,9 @@ import { afterEach, expect, test } from "vitest";
import {
resolveBinding, resolveEvidenceBinding, resolveRuntimeBindings, supportsSessionRuntime,
} from "../src/workspaces/bindings.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const workspaceV4 = parseWorkspaceYaml(`workspace:
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -63,7 +63,7 @@ test("requires workspace-v4 REST credentials unless the DWH diagnostic declares
THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
}, []).missing).toContain("THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE");
const noAuth = parseWorkspaceYaml(`workspace:
const noAuth = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: No auth
@@ -89,7 +89,7 @@ diagnostics:
test("rejects unsupported transports and secret paths outside configured roots", () => {
const outside = secretPath("outside-password");
const allowed = secretPath("allowed-password");
const directOnly = parseWorkspaceYaml(`workspace:
const directOnly = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Direct only
@@ -130,7 +130,7 @@ test("runtime bindings contain only DWH and Evidence roles", () => {
});
function withEvidence(source: Record<string, unknown>) {
return parseWorkspaceYaml(`workspace:
return parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
+2 -2
View File
@@ -1,12 +1,12 @@
import { expect, test } from "vitest";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
import {
CATALOG_PATH,
assertCatalogMatchesDescriptor,
parseWorkspaceCatalogYaml,
} from "../src/workspaces/catalog.js";
const descriptor = parseWorkspaceYaml(`workspace:
const descriptor = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd
name: Policlinico San Donato
+9 -9
View File
@@ -3,9 +3,9 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { buildInstallationContract, renderWorkspaceDocs } from "../src/workspaces/contracts.js";
import { type CanonicalWorkspace, parseWorkspaceYaml } from "../src/workspaces/schema.js";
import { type CanonicalWorkspace, parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
const workspaceV4 = parseWorkspaceYaml(`workspace:
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -38,7 +38,7 @@ test.each([
["rest_api", "BASE_URL", "HOST"],
["ssh_tunnel", "SSH_PRIVATE_KEY_FILE", "BASE_URL"],
] as const)("documents only the DWH fields for %s", (transport, included, excluded) => {
const descriptor = parseWorkspaceYaml(renderWorkspaceWithoutEvidence()
const descriptor = parseRuntimeWorkspaceYaml(renderWorkspaceWithoutEvidence()
.replace("[postgres_direct]", `[${transport}]`));
const variables = buildInstallationContract(descriptor).variables;
expect(variables.find(({ suffix }) => suffix === included)?.transports).toEqual([transport]);
@@ -87,7 +87,7 @@ test.each([
],
},
])("generates source-specific $mode Evidence file bindings", ({ source, expected }) => {
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
const contract = buildInstallationContract(descriptor);
const evidence = contract.variables.filter((variable) => variable.role === "EVIDENCE");
@@ -102,7 +102,7 @@ test.each([
{ type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" },
{ type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" },
])("omits Evidence installation variables for $type modes without file credentials", (source) => {
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
expect(buildInstallationContract(descriptor).variables.some((variable) => variable.role === "EVIDENCE"))
.toBe(false);
});
@@ -151,7 +151,7 @@ const evidenceSources = [
] as const;
test.each(evidenceSources)("renders deterministic public Evidence docs for $label", ({ source, variables }) => {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
);
const firstContract = buildInstallationContract(descriptor);
@@ -178,7 +178,7 @@ test.each(evidenceSources)("renders deterministic public Evidence docs for $labe
});
test("documents the S3 session token file as optional", () => {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:
source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files }
`,
@@ -197,7 +197,7 @@ test("documents the S3 session token file as optional", () => {
});
test("documents same-revision filesystem ownership without claiming P1 materialization", () => {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:\n source: { type: filesystem, uri: psd-clinical/evidence }\n`,
);
const docs = renderWorkspaceDocs(descriptor).markdown;
@@ -236,7 +236,7 @@ test.each([
const previous = process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE;
process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = binding;
try {
const descriptor = parseWorkspaceYaml(
const descriptor = parseRuntimeWorkspaceYaml(
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
);
const generated = JSON.stringify({
+15 -10
View File
@@ -9,9 +9,9 @@ import {
type DiagnosticAdapters,
} from "../src/workspaces/diagnostics.js";
import type { RuntimeBindings } from "../src/workspaces/runtime-renderer.js";
import { parseWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
import { parseRuntimeWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
const workspace = parseWorkspaceYaml(`workspace:
const workspace = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
@@ -81,9 +81,11 @@ test("diagnoses workspace-v4 DWH plus installation-derived Qdrant and Ollama", a
role: "dwh", transport: "postgres_direct",
resource: { database: "warehouse", schema: "datawarehouse" },
}));
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({
baseUrl: "http://qdrant:6333", collection: "psd-clinical",
}));
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
expect(vi.mocked(adapters.inspectQdrant).mock.calls.map(([request]) => request.collection)).toEqual([
"psd-clinical-reference",
"psd-clinical-memory",
]);
expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({
baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b",
}));
@@ -107,14 +109,14 @@ test("can delegate the DWH probe to Database Management", async () => {
}],
});
expect(adapters.probeConnector).not.toHaveBeenCalled();
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(1);
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
expect(adapters.probeEmbedding).toHaveBeenCalledTimes(1);
});
test("reports incompatible internal Qdrant or Ollama metadata", async () => {
const vector = await diagnose(successfulAdapters({
inspectQdrant: vi.fn(async () => ({
collection: "psd-clinical", dimensions: 768, distance: "cosine",
inspectQdrant: vi.fn(async (request) => ({
collection: request.collection, dimensions: 768, distance: "cosine",
})),
}))(workspace, bindings, { writeProbe: false });
expect(vector.activatable).toBe(false);
@@ -163,7 +165,7 @@ test("keeps workspace-v4 DWH SSH diagnostic-only and runtime-inactive", async ()
});
test("uses the workspace-v4 declared DWH REST diagnostic and auth policy", async () => {
const restWorkspace = parseWorkspaceYaml(`workspace:
const restWorkspace = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: REST workspace
@@ -428,7 +430,7 @@ test("uses a REST secret only as a header and redacts it from failed diagnostics
const credentialFile = join(root, "api-key");
const canary = "CANARY-REST-AUTH-SECRET";
await writeFile(credentialFile, canary);
const restDescriptor = parseWorkspaceYaml(`workspace:
const restDescriptor = parseRuntimeWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: REST auth
@@ -548,6 +550,9 @@ test("returns observed normalized Qdrant distance for semantic mismatch classifi
test("classifies an observed non-cosine Qdrant distance as semantic incompatibility", async () => {
const fetchMock = vi.fn()
.mockResolvedValueOnce(new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
}), { status: 200 }))
.mockResolvedValueOnce(new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
}), { status: 200 }))
@@ -16,11 +16,6 @@ const validYaml = `workspace:
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
`;
const runFile = promisify(execFile);
+10 -23
View File
@@ -15,16 +15,6 @@ export const validYaml = `workspace:
name: Policlinico San Donato
description: Clinical data warehouse workspace
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
port: 5432
timeout_ms: 5000
supported_transports:
- postgres_direct
- rest_api
- ssh_tunnel
`;
test("rejects unknown keys and invalid immutable IDs", () => {
@@ -41,21 +31,18 @@ test("rejects executable or otherwise custom YAML tags in canonical descriptors"
))).toThrow(/tag|yaml/i);
});
test("accepts optional connection ports and timeouts but rejects unsafe values", () => {
expect(parseWorkspaceYaml(validYaml).dwh.port).toBe(5432);
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 0")))
.toThrow(/port/i);
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 65536")))
.toThrow(/port/i);
expect(() => parseWorkspaceYaml(validYaml.replace("timeout_ms: 5000", "timeout_ms: 0")))
.toThrow(/timeout/i);
test("rejects database configuration and diagnostics in authored workspace YAML", () => {
expect(() => parseWorkspaceYaml(`${validYaml}dwh:\n engine: postgres\n database: d\n schema: s\n supported_transports: [postgres_direct]\n`))
.toThrow(/must not contain database configuration/i);
expect(() => parseWorkspaceYaml(`${validYaml}diagnostics:\n dwh_rest:\n method: GET\n path: \/health\n auth: none\n`))
.toThrow(/must not contain database configuration/i);
});
test("accepts a model-free schema v4 workspace", () => {
test("accepts a database-free schema v4 workspace", () => {
expect(parseWorkspaceYaml(validYaml)).toMatchObject({
workspace: { schema_version: 4, id: "psd-clinical" },
dwh: { database: "postgres", schema: "datawarehouse" },
});
expect(parseWorkspaceYaml(validYaml)).not.toHaveProperty("dwh");
});
test("committed example descriptors parse as exact schema v4 workspaces", () => {
@@ -137,7 +124,7 @@ llm_policy:
- zai/glm-5.2
`],
])("rejects schema %s descriptors at parser and object-validator boundaries", (_version, yaml) => {
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|4/i);
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|database configuration|4/i);
expect(() => validateWorkspaceDescriptor(parse(yaml))).toThrow(/schema_version|invalid literal|4/i);
});
@@ -159,7 +146,7 @@ test("constructs diagnostic URLs only when the resolved URL remains on the servi
)).toThrow(/origin/i);
});
test("rejects REST diagnostic declarations without their matching connector transport", () => {
test("rejects REST diagnostic declarations in authored descriptors", () => {
const diagnostics = `diagnostics:
dwh_rest:
method: POST
@@ -171,7 +158,7 @@ test("rejects REST diagnostic declarations without their matching connector tran
`;
const declared = `${validYaml}${diagnostics}`;
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", ""))).toThrow(/dwh_rest/i);
expect(() => parseWorkspaceYaml(declared)).toThrow(/database configuration/i);
});
test("serializes canonical YAML that parses back to the same workspace", () => {
@@ -138,6 +138,30 @@ test("projects aggregate no-Evidence and completed-stage outcomes without rerunn
expect(deps.persistJob).not.toHaveBeenCalled();
});
test("complete preprocessing preflights BM25 before continuing with Evidence", async () => {
const deps = dependencies();
deps.evidencePreflight.mockResolvedValue({
ok: false as const,
code: "semantic_index_incompatible" as const,
});
const state = job({ completedStages: ["catalog_snapshot", "schema_index"] });
const result = await continueEvidencePreprocessing(
{ evidence: filesystemEvidence, job: state },
deps,
);
expect(deps.evidencePreflight).toHaveBeenCalledOnce();
expect(deps.runStage).not.toHaveBeenCalled();
expect(result).toEqual({
status: "failed",
code: "semantic_index_incompatible",
runId: "a".repeat(32),
childRuns: {},
completedStages: ["catalog_snapshot", "schema_index"],
});
});
test("preserves the narrow standalone projection for an already completed Evidence stage", async () => {
const deps = dependencies();