feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s
Publish documentation / publish (push) Successful in 2m12s
This commit is contained in:
@@ -97,31 +97,23 @@ const fleetSnapshot: ObservedSchemaSnapshot = {
|
||||
}],
|
||||
};
|
||||
|
||||
test("lists every YAML workspace and creates its one database configuration", async () => {
|
||||
test("lists every workspace and creates its Catalog database configuration", async () => {
|
||||
const { app, secretStore } = setup();
|
||||
const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(initial.statusCode).toBe(200);
|
||||
expect(initial.json()).toMatchObject([{
|
||||
workspaceId: "psd-clinical",
|
||||
configured: false,
|
||||
databaseName: "warehouse",
|
||||
databaseName: "",
|
||||
workspaceRevision: { commit: revision.commit, blob: revision.blob },
|
||||
workspaceEvidence: { sourceType: null, state: "not_declared" },
|
||||
runtimeBinding: {
|
||||
transport: "postgres_direct",
|
||||
configurationState: "configuration_required",
|
||||
sessionTransportSupported: true,
|
||||
},
|
||||
runtimeBinding: null,
|
||||
}]);
|
||||
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
|
||||
secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" });
|
||||
const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(runtimeReady.json()).toMatchObject([{
|
||||
runtimeBinding: { configurationState: "ready", sessionTransportSupported: true },
|
||||
runtimeBinding: null,
|
||||
}]);
|
||||
|
||||
const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
|
||||
@@ -166,7 +158,7 @@ test("projects remote Evidence credential state without conflating catalog secre
|
||||
}]);
|
||||
});
|
||||
|
||||
test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
|
||||
test("lists orphaned records and keeps the REST diagnostic path in the Catalog", async () => {
|
||||
const { app, repository } = setup();
|
||||
await repository.create({
|
||||
workspaceId: "removed-workspace",
|
||||
@@ -186,7 +178,7 @@ test("lists orphaned records and takes the REST diagnostic path from workspace Y
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(201);
|
||||
expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
|
||||
expect(created.json()).toMatchObject({ binding: { restPath: "/client-controlled" } });
|
||||
|
||||
const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
|
||||
expect(rows).toEqual(expect.arrayContaining([
|
||||
|
||||
@@ -577,7 +577,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
|
||||
expect.objectContaining({
|
||||
sequence: 3,
|
||||
level: "info",
|
||||
message: `Generated description for Catalog Column ${column.id}.`,
|
||||
message: 'Generated description for Column "patients.birth_date".',
|
||||
}),
|
||||
expect.objectContaining({ sequence: 4, level: "info", message: "Description generation completed." }),
|
||||
]);
|
||||
@@ -934,7 +934,10 @@ test("generates selected Catalog Columns in caller order through sequential batc
|
||||
const events = await repository.listDescriptionGenerationEvents(run.id);
|
||||
expect(events.map((event) => event.sequence)).toEqual(Array.from({ length: 14 }, (_, index) => index + 1));
|
||||
expect(events.slice(2, -1).map((event) => event.message)).toEqual(
|
||||
orderedIds.map((targetId) => `Generated description for Catalog Column ${targetId}.`),
|
||||
orderedIds.map((targetId) => {
|
||||
const target = columns.find((column) => column.id === targetId)!;
|
||||
return `Generated description for Column "patients.${target.name}".`;
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
pending[0]!.resolve(responseFor(orderedIds.slice(0, 10), 0));
|
||||
@@ -1803,7 +1806,9 @@ test("retains completed batch writes when a later batch response is malformed",
|
||||
});
|
||||
const events = await repository.listDescriptionGenerationEvents(run.id);
|
||||
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
|
||||
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
|
||||
originalColumns.slice(0, 10).map(
|
||||
(target) => `Generated description for Column "patients.${target.name}".`,
|
||||
),
|
||||
);
|
||||
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
|
||||
expect.objectContaining({
|
||||
@@ -1812,7 +1817,7 @@ test("retains completed batch writes when a later batch response is malformed",
|
||||
);
|
||||
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
|
||||
message: `The model response did not match the required schema. Affected target: Column "patients.${originalColumns[10]!.name}".`,
|
||||
}));
|
||||
} finally {
|
||||
await app.close();
|
||||
@@ -2301,7 +2306,7 @@ test("generates selected Catalog Tables with structural column context and local
|
||||
expect((await repository.listDescriptionGenerationEvents(run.id)).map((event) => event.message)).toEqual([
|
||||
"Description generation queued.",
|
||||
"Description generation started.",
|
||||
`Stored non-generatable result for Catalog Table ${table.id}.`,
|
||||
'Stored non-generatable result for Table "patients".',
|
||||
"Description generation completed.",
|
||||
]);
|
||||
} finally {
|
||||
@@ -2445,7 +2450,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
|
||||
expect(`${JSON.stringify(run)}${events.body}`).not.toContain(sensitiveDiagnostic);
|
||||
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model provider request failed. Affected Catalog Column target: ${column.id}.`,
|
||||
message: 'The model provider request failed. Affected target: Column "patients.birth_date".',
|
||||
}));
|
||||
} finally {
|
||||
await app.close();
|
||||
@@ -2536,7 +2541,7 @@ test.each([
|
||||
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
|
||||
expect.objectContaining({
|
||||
level: "error",
|
||||
message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
|
||||
message: `${failureMessage} Affected targets: Column "patients.first_column", Column "patients.second_column".`,
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
|
||||
@@ -13,10 +13,12 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
|
||||
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
||||
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
||||
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
||||
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
|
||||
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
||||
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
||||
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
|
||||
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
|
||||
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
|
||||
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
@@ -52,10 +54,12 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
||||
await upSensitiveDataFlag(db);
|
||||
await upDescriptionGeneration(db);
|
||||
await upSensitiveSuggestionRuns(db);
|
||||
await upLogicalRelationships(db);
|
||||
await upAiTokenUsage(db);
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
await upSensitivityReason(db);
|
||||
await upCatalogPreprocessingState(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -282,7 +286,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
||||
expect(events.statusCode).toBe(200);
|
||||
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
|
||||
message: 'The model provider request failed. Affected target: Column "patients.status".',
|
||||
}));
|
||||
expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usa
|
||||
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
||||
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
|
||||
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
|
||||
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
@@ -58,6 +59,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
await upSensitivityReason(db);
|
||||
await upCatalogPreprocessingState(db);
|
||||
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
|
||||
.select(["engine", "modelId", "policyVersion", "unknown"])
|
||||
.where("id", "=", historicalSuggestionRunId)
|
||||
@@ -232,6 +234,77 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
expect(await repository.listSyncRuns(created.id)).toEqual([
|
||||
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
||||
]);
|
||||
|
||||
const lockedDatabase = await repository.create({
|
||||
workspaceId: "preprocessing-lock",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "public",
|
||||
binding: {
|
||||
transport: "postgres_direct", host: "lock.internal", port: 5432, username: "reader",
|
||||
},
|
||||
});
|
||||
await repository.applySchemaSync(
|
||||
lockedDatabase.id,
|
||||
lockedDatabase.version,
|
||||
"all",
|
||||
[],
|
||||
{
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [], columns: [], relationships: [],
|
||||
},
|
||||
);
|
||||
const beforePreprocessing = (await repository.get(lockedDatabase.id))!;
|
||||
const preprocessing = await repository.beginPreprocessing(
|
||||
"preprocessing-lock",
|
||||
"sha256:" + "1".repeat(64),
|
||||
);
|
||||
expect(preprocessing).toMatchObject({ kind: "started" });
|
||||
await expect(db.insertInto("catalogTables").values({
|
||||
id: randomUUID(),
|
||||
databaseId: lockedDatabase.id,
|
||||
name: "blocked_write",
|
||||
sourceComment: null,
|
||||
description: null,
|
||||
generatedDescription: null,
|
||||
}).execute()).rejects.toThrow("catalog preprocessing is running");
|
||||
await expect(repository.createDescriptionGenerationRun(
|
||||
lockedDatabase.id,
|
||||
"all",
|
||||
"openai/gpt-5-mini",
|
||||
"en",
|
||||
0,
|
||||
)).rejects.toThrow("catalog preprocessing is running");
|
||||
await repository.recordTest(lockedDatabase.id, lockedDatabase.version, {
|
||||
connectionStatus: "reachable",
|
||||
testedVersion: lockedDatabase.version,
|
||||
lastTestedAt: "2026-01-01T00:00:00Z",
|
||||
});
|
||||
expect((await repository.get(lockedDatabase.id))?.metadataContentRevision)
|
||||
.toBe(beforePreprocessing.metadataContentRevision);
|
||||
await expect(repository.finishPreprocessing(
|
||||
"preprocessing-lock",
|
||||
beforePreprocessing.metadataContentRevision,
|
||||
"sha256:" + "1".repeat(64),
|
||||
{ status: "succeeded" },
|
||||
)).resolves.toMatchObject({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: beforePreprocessing.metadataContentRevision,
|
||||
});
|
||||
await db.insertInto("catalogTables").values({
|
||||
id: randomUUID(),
|
||||
databaseId: lockedDatabase.id,
|
||||
name: "allowed_after_preprocessing",
|
||||
sourceComment: null,
|
||||
description: null,
|
||||
generatedDescription: null,
|
||||
}).execute();
|
||||
await expect(repository.get(lockedDatabase.id)).resolves.toMatchObject({
|
||||
preprocessingStatus: "failed",
|
||||
metadataContentRevision: beforePreprocessing.metadataContentRevision + 1,
|
||||
});
|
||||
expect(await repository.delete(lockedDatabase.id, lockedDatabase.version)).toBe(true);
|
||||
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
|
||||
expect(await repository.delete(created.id, 2)).toBe(true);
|
||||
expect(await repository.list()).toEqual([]);
|
||||
@@ -355,7 +428,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected and database-wide descriptions", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
@@ -449,6 +522,22 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel
|
||||
description: "Still curated visits",
|
||||
generatedDescription: "Generated visits",
|
||||
});
|
||||
|
||||
expect(await repository.consolidateGeneratedDescriptions(
|
||||
database.id, "database", [],
|
||||
)).toEqual({ copied: 3, skipped: 1 });
|
||||
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
||||
description: "Generated visits",
|
||||
generatedDescription: "Generated visits",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
||||
description: "Generated id",
|
||||
generatedDescription: "Generated id",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
||||
description: "Keep patient reference",
|
||||
generatedDescription: null,
|
||||
});
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
@@ -473,6 +562,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
await upSensitivityReason(db);
|
||||
await upCatalogPreprocessingState(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const firstDatabase = await repository.create({
|
||||
workspaceId: "generation-one",
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
|
||||
import { resolveCatalogRuntimeBinding } from "../src/catalog/runtime-binding.js";
|
||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("projects a Catalog database into a runtime without database data in workspace YAML", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "thoth-catalog-runtime-"));
|
||||
roots.push(root);
|
||||
const secretStore = new WorkspaceSecretStore({
|
||||
root: join(root, "vault"),
|
||||
runtimeRoot: join(root, "runtime"),
|
||||
installationId: "test",
|
||||
});
|
||||
secretStore.putMany("sales", {
|
||||
"catalog.dwh.password": "catalog-password",
|
||||
"evidence.signed_urls": '["https://signed.example.test/evidence"]',
|
||||
});
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: {
|
||||
schema_version: 4,
|
||||
id: "sales",
|
||||
name: "Sales",
|
||||
language: "en",
|
||||
},
|
||||
evidence: {
|
||||
schema_version: 1,
|
||||
source: {
|
||||
type: "http",
|
||||
uris: ["https://evidence.example.test/guide.md"],
|
||||
authentication: "signed_urls_file",
|
||||
connect_timeout_ms: 1_000,
|
||||
read_timeout_ms: 5_000,
|
||||
max_bytes: 10_000,
|
||||
max_redirects: 2,
|
||||
allow_private_hosts: false,
|
||||
max_cache_bytes: 20_000,
|
||||
},
|
||||
policy: { max_chunk_chars: 4_000, retain_published_generations: 1 },
|
||||
},
|
||||
};
|
||||
const database: WorkspaceDatabase = {
|
||||
id: "database-1",
|
||||
workspaceId: "sales",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "analytics",
|
||||
version: 3,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
binding: {
|
||||
transport: "postgres_direct",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
connectionStatus: "reachable",
|
||||
metadataContentRevision: 7,
|
||||
preprocessingStatus: "failed",
|
||||
};
|
||||
|
||||
const lease = resolveCatalogRuntimeBinding({
|
||||
workspace,
|
||||
database,
|
||||
environment: {},
|
||||
secretRoots: [],
|
||||
secretStore,
|
||||
});
|
||||
const passwordPath = lease.bindings.dwh.values.THT_WS_SALES_DWH_PASSWORD_FILE;
|
||||
const evidencePath = lease.bindings.evidence.values.THT_WS_SALES_EVIDENCE_SIGNED_URLS_FILE;
|
||||
try {
|
||||
expect(workspace.dwh).toBeUndefined();
|
||||
expect(lease.workspace.dwh).toMatchObject({
|
||||
database: "warehouse",
|
||||
schema: "analytics",
|
||||
supported_transports: ["postgres_direct"],
|
||||
});
|
||||
expect(lease.bindings.dwh).toMatchObject({
|
||||
transport: "postgres_direct",
|
||||
missing: [],
|
||||
values: {
|
||||
THT_WS_SALES_DWH_HOST: "db.internal",
|
||||
THT_WS_SALES_DWH_PORT: "5432",
|
||||
THT_WS_SALES_DWH_USER: "reader",
|
||||
},
|
||||
});
|
||||
expect(readFileSync(passwordPath, "utf8")).toBe("catalog-password");
|
||||
expect(readFileSync(evidencePath, "utf8")).toContain("signed.example.test");
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
expect(existsSync(passwordPath)).toBe(false);
|
||||
expect(existsSync(evidencePath)).toBe(false);
|
||||
});
|
||||
@@ -394,12 +394,19 @@ test("consolidates non-empty generated column descriptions and preserves curated
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("consolidates generated descriptions for every column in a database", async () => {
|
||||
test("consolidates generated descriptions for every table and column in a database", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const tables = await repository.listTables(database.id);
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
await repository.updateTableMetadata(
|
||||
database.id,
|
||||
patients.id,
|
||||
patients.version,
|
||||
"Curated patients",
|
||||
"Generated patients",
|
||||
);
|
||||
const patientId = (await repository.listColumns(database.id, patients.id))[0]!;
|
||||
const visitColumns = await repository.listColumns(database.id, visits.id);
|
||||
const visitId = visitColumns.find((column) => column.name === "id")!;
|
||||
@@ -432,11 +439,16 @@ test("consolidates generated descriptions for every column in a database", async
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "database_columns" },
|
||||
payload: { target: "database" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({ copied: 2, skipped: 1 });
|
||||
expect(response.json()).toEqual({ copied: 3, skipped: 2 });
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
||||
description: "Generated patients",
|
||||
generatedDescription: "Generated patients",
|
||||
version: patients.version + 2,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({
|
||||
description: "Generated patient identifier",
|
||||
generatedDescription: "Generated patient identifier",
|
||||
@@ -529,6 +541,11 @@ test("strictly validates description consolidation database and target ids", asy
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "tables", targetIds: [table.id], unexpected: true },
|
||||
}),
|
||||
app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "database", targetIds: [table.id] },
|
||||
}),
|
||||
app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
@@ -536,7 +553,7 @@ test("strictly validates description consolidation database and target ids", asy
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400]);
|
||||
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400, 400]);
|
||||
for (const response of responses) {
|
||||
expect(response.json()).toEqual({
|
||||
code: "description_consolidation_invalid",
|
||||
|
||||
@@ -41,7 +41,10 @@ function directRendered(): Record<string, unknown> {
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
base_url: "http://qdrant:6333",
|
||||
collection: "psd-clinical",
|
||||
collections: {
|
||||
reference: "psd-clinical-reference",
|
||||
memory: "psd-clinical-memory",
|
||||
},
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
collection_lifecycle: "require_existing",
|
||||
@@ -103,10 +106,10 @@ function restRendered(): Record<string, unknown> {
|
||||
}
|
||||
|
||||
const directCanonical =
|
||||
`{"schemaVersion":2,"dwh":{` +
|
||||
`{"schemaVersion":3,"dwh":{` +
|
||||
`"engine":"postgres","database":"postgres","schema":"datawarehouse",` +
|
||||
`"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` +
|
||||
`"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` +
|
||||
`"vector":{"collections":{"reference":"psd-clinical-reference","memory":"psd-clinical-memory"},"dimensions":1024,"distance":"cosine"},` +
|
||||
`"embedding":{"id":"ollama/qwen3-embedding:0.6b","model":"qwen3-embedding:0.6b","dimensions":1024},` +
|
||||
`"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` +
|
||||
`"indexes":"/data/sessions/psd-clinical/indexes"}}`;
|
||||
@@ -128,7 +131,7 @@ test("canonical effective config excludes secrets, evidence, session storage, an
|
||||
expect(json).not.toContain("sources");
|
||||
expect(json).not.toContain("collection_lifecycle");
|
||||
expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity
|
||||
expect(json).not.toContain("memory");
|
||||
expect(json).not.toContain('"memory":"/data');
|
||||
expect(json).not.toContain('"sessions"');
|
||||
});
|
||||
|
||||
@@ -190,7 +193,16 @@ test("DWH-affecting changes alter the effective config identity", () => {
|
||||
const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } };
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore);
|
||||
|
||||
const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record<string, any>).vector, collection: "other" } } };
|
||||
const changedCollection = {
|
||||
...base,
|
||||
resources: {
|
||||
...base.resources,
|
||||
vector: {
|
||||
...(base.resources as Record<string, any>).vector,
|
||||
collections: { reference: "other-reference", memory: "other-memory" },
|
||||
},
|
||||
},
|
||||
};
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore);
|
||||
|
||||
const changedEmbeddingIdentity = { ...base, resources: { ...base.resources, embeddings: { ...(base.resources as Record<string, any>).embeddings, model: "other-embedding" } } };
|
||||
|
||||
@@ -19,11 +19,6 @@ const validYaml = `workspace:
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
async function git(cwd: string, args: string[]): Promise<string> {
|
||||
|
||||
@@ -24,11 +24,6 @@ const descriptor = `workspace:
|
||||
id: research
|
||||
name: Research
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: filesystem
|
||||
|
||||
@@ -12,6 +12,7 @@ import { PiProcessManager } from "../src/pi/pi-process-manager.js";
|
||||
import { validateDeclarativePiConfig } from "../src/pi/managed-config.js";
|
||||
import Fastify from "fastify";
|
||||
import { sessionRoutes } from "../src/routes/sessions.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
|
||||
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
|
||||
const SCRIPT = path.resolve("../harness/tests/fake_pi/scripts/f1_disambiguation.json");
|
||||
@@ -517,7 +518,7 @@ test("creates a session from the active immutable workspace revision", async ()
|
||||
}));
|
||||
});
|
||||
|
||||
test("hands one effective relationship snapshot to both retrieval and Pi", async () => {
|
||||
test("hands one Catalog-backed runtime to both retrieval and Pi", async () => {
|
||||
const effective = JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
workspaceId: "default",
|
||||
@@ -556,10 +557,9 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(render).toHaveBeenCalledWith("default");
|
||||
expect(render).not.toHaveBeenCalled();
|
||||
expect(acquireWorkspaceRuntime).toHaveBeenCalledWith(
|
||||
expect.stringContaining("/default.yaml"),
|
||||
effective,
|
||||
);
|
||||
expect(createFor).toHaveBeenCalledWith(
|
||||
"effective-map",
|
||||
@@ -574,27 +574,103 @@ test("hands one effective relationship snapshot to both retrieval and Pi", async
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects an SSH-only workspace before persisting or starting a session", async () => {
|
||||
test("refuses core admission when the workspace preprocessing fingerprint is stale", async () => {
|
||||
const sessionNew = vi.fn();
|
||||
const workspaceInputFingerprint = vi.fn(async () => "sha256:current");
|
||||
const revision = {
|
||||
id: "default",
|
||||
commit: "a".repeat(40),
|
||||
blob: "b".repeat(40),
|
||||
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/default.yaml`,
|
||||
};
|
||||
const catalogRepository = new MemoryCatalogRepository();
|
||||
const database = await catalogRepository.create({
|
||||
workspaceId: "default",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "public",
|
||||
binding: {
|
||||
transport: "postgres_direct",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
});
|
||||
await catalogRepository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [],
|
||||
columns: [],
|
||||
relationships: [],
|
||||
});
|
||||
const started = await catalogRepository.beginPreprocessing("default", "sha256:previous");
|
||||
expect(started.kind).toBe("started");
|
||||
await catalogRepository.finishPreprocessing(
|
||||
"default",
|
||||
0,
|
||||
"sha256:previous",
|
||||
{ status: "succeeded" },
|
||||
);
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: { sessionNew, workspaceInputFingerprint } as any,
|
||||
catalogRepository,
|
||||
workspaceRegistry: {
|
||||
acquireSessionRevision: async () => ({
|
||||
workspace: operationalWorkspace("default"),
|
||||
revision,
|
||||
abort: async () => {},
|
||||
markPersisted: async () => {},
|
||||
}),
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "default", thinking: "low" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/sessions",
|
||||
payload: { question: "q", workspaceId: "default" },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toMatchObject({ code: "preprocessing_required" });
|
||||
expect(workspaceInputFingerprint).toHaveBeenCalledWith(revision.snapshotPath);
|
||||
expect(sessionNew).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("rejects an SSH-only Catalog binding before persisting or starting a session", async () => {
|
||||
const sessionNew = vi.fn(async () => ({ id: "must-not-exist" }));
|
||||
const workspaceInputFingerprint = vi.fn(async () => "sha256:unused");
|
||||
const ensure = vi.fn(async () => ({ ok: true }));
|
||||
const createFor = vi.fn();
|
||||
const abort = vi.fn(async () => {});
|
||||
const markPersisted = vi.fn(async () => {});
|
||||
const catalogRepository = new MemoryCatalogRepository();
|
||||
await catalogRepository.create({
|
||||
workspaceId: "ssh-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "postgres",
|
||||
schema: "public",
|
||||
binding: {
|
||||
transport: "ssh_tunnel",
|
||||
username: "reader",
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 22,
|
||||
sshUsername: "tunnel",
|
||||
sshTargetHost: "postgres.internal",
|
||||
sshTargetPort: 5432,
|
||||
},
|
||||
});
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: { sessionNew, searchPack: async () => {} } as any,
|
||||
thtRunner: { sessionNew, searchPack: async () => {}, workspaceInputFingerprint } as any,
|
||||
catalogRepository,
|
||||
readiness: { ensure } as any,
|
||||
mgr: { get: () => undefined, createFor } as any,
|
||||
getSettings: () => ({ workspace: "ssh-workspace" }) as any,
|
||||
workspaceRuntimeSupport: vi.fn(() => false),
|
||||
workspaceRuntimeSupport: vi.fn(() => true),
|
||||
workspaceRegistry: {
|
||||
acquireSessionRevision: vi.fn(async () => ({
|
||||
workspace: {
|
||||
workspace: { schema_version: 4, id: "ssh-workspace", name: "SSH", language: "en" },
|
||||
dwh: {
|
||||
engine: "postgres", database: "postgres", schema: "public",
|
||||
supported_transports: ["ssh_tunnel"],
|
||||
},
|
||||
},
|
||||
workspace: operationalWorkspace("ssh-workspace"),
|
||||
revision: {
|
||||
id: "ssh-workspace", commit: "a".repeat(40), blob: "b".repeat(40),
|
||||
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/ssh-workspace.yaml`,
|
||||
@@ -610,6 +686,7 @@ test("rejects an SSH-only workspace before persisting or starting a session", as
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toMatchObject({ code: "workspace_not_activatable" });
|
||||
expect(ensure).not.toHaveBeenCalled();
|
||||
expect(workspaceInputFingerprint).not.toHaveBeenCalled();
|
||||
expect(sessionNew).not.toHaveBeenCalled();
|
||||
expect(createFor).not.toHaveBeenCalled();
|
||||
expect(abort).toHaveBeenCalledOnce();
|
||||
|
||||
@@ -500,8 +500,9 @@ async function createRealRouteFixture() {
|
||||
await git(author, ["init", "--initial-branch=main"]);
|
||||
await git(author, ["config", "user.name", "Workspace Route Test"]);
|
||||
await git(author, ["config", "user.email", "workspace-route@example.invalid"]);
|
||||
const { dwh: _runtimeDwh, diagnostics: _runtimeDiagnostics, ...authoredWorkspace } = workspace;
|
||||
const descriptor: CanonicalWorkspace = {
|
||||
...workspace,
|
||||
...authoredWorkspace,
|
||||
evidence: {
|
||||
source: {
|
||||
type: "filesystem",
|
||||
|
||||
@@ -52,8 +52,11 @@ test("Qdrant readiness uses only the internal URL and accepts the exact collecti
|
||||
const request = vi.fn(async () => response(200, collection()));
|
||||
|
||||
await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ ok: true, state: "ready" });
|
||||
expect(request).toHaveBeenCalledOnce();
|
||||
expect(request.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd");
|
||||
expect(request).toHaveBeenCalledTimes(2);
|
||||
expect(request.mock.calls.map((call) => call[0])).toEqual([
|
||||
"http://qdrant:6333/collections/psd-reference",
|
||||
"http://qdrant:6333/collections/psd-memory",
|
||||
]);
|
||||
expect(request.mock.calls[0][1]).toMatchObject({ method: "GET", signal: expect.any(AbortSignal) });
|
||||
});
|
||||
|
||||
|
||||
@@ -30,11 +30,11 @@ function ok(operation: string): WorkspaceOperationResult {
|
||||
};
|
||||
}
|
||||
|
||||
test("entrypoint emits exactly one pristine JSON document and maps success/block/failure exits", async () => {
|
||||
test("entrypoint emits exactly one pristine JSON document and maps success/failure exits", async () => {
|
||||
const service = {
|
||||
inspect: vi.fn(async () => ok("inspect")),
|
||||
preprocessDwh: vi.fn(async () => ({ ...ok("preprocess dwh"), status: "blocked", code: "manual_review_required" as const })),
|
||||
run: vi.fn(async () => ({ ...ok("preprocess run"), status: "failed", code: "semantic_index_incompatible" as const })),
|
||||
clear: vi.fn(async () => ok("preprocess clear")),
|
||||
} as any;
|
||||
|
||||
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
@@ -42,13 +42,13 @@ test("entrypoint emits exactly one pristine JSON document and maps success/block
|
||||
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "inspect", code: "ok" });
|
||||
expect(inspectIo.stderr.join("")).toBe("");
|
||||
|
||||
const blockedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-dwh"], service, blockedIo)).toBe(3);
|
||||
expect(JSON.parse(blockedIo.stdout.join(""))).toMatchObject({ code: "manual_review_required" });
|
||||
|
||||
const failedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-run"], service, failedIo)).toBe(1);
|
||||
expect(JSON.parse(failedIo.stdout.join(""))).toMatchObject({ code: "semantic_index_incompatible" });
|
||||
|
||||
const clearIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-clear"], service, clearIo)).toBe(0);
|
||||
expect(JSON.parse(clearIo.stdout.join(""))).toMatchObject({ operation: "preprocess clear", code: "ok" });
|
||||
});
|
||||
|
||||
test("malformed stdin, unknown commands, and extra fields fail with exit 2 but still return bounded JSON", async () => {
|
||||
@@ -84,31 +84,19 @@ test("raw exception text is redacted from stderr and stdout remains within the p
|
||||
expect(captured.stderr.join("")).not.toContain("SELECT *");
|
||||
});
|
||||
|
||||
test("vector-inspect and vector-rebuild dispatch to the service with the exact envelope", async () => {
|
||||
test("retired partial preprocessing commands are rejected without dispatch", async () => {
|
||||
const service = {
|
||||
vectorInspect: vi.fn(async () => ok("vector inspect")),
|
||||
vectorRebuild: vi.fn(async () => ok("vector rebuild")),
|
||||
preprocessDwh: vi.fn(),
|
||||
vectorInspect: vi.fn(),
|
||||
vectorRebuild: vi.fn(),
|
||||
} as any;
|
||||
|
||||
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-inspect"], service, inspectIo)).toBe(0);
|
||||
expect(service.vectorInspect).toHaveBeenCalledWith({ workspaceId: "psd-clinical" });
|
||||
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "vector inspect", code: "ok" });
|
||||
|
||||
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(0);
|
||||
expect(service.vectorRebuild).toHaveBeenCalledWith({
|
||||
workspaceId: "psd-clinical",
|
||||
collection: "psd-clinical",
|
||||
confirm: "psd-clinical",
|
||||
destroy: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("vector-rebuild without exact confirmation is refused by the service", async () => {
|
||||
const service = {
|
||||
vectorRebuild: vi.fn(async () => ({ ...ok("vector rebuild"), status: "failed", code: "semantic_index_incompatible" as const })),
|
||||
} as any;
|
||||
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "other", confirm: "other", destroy: true }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(1);
|
||||
for (const command of ["preprocess-dwh", "vector-inspect", "vector-rebuild"]) {
|
||||
const captured = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
|
||||
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", command], service, captured)).toBe(2);
|
||||
expect(JSON.parse(captured.stdout.join(""))).toMatchObject({ status: "failed", operation: command });
|
||||
}
|
||||
expect(service.preprocessDwh).not.toHaveBeenCalled();
|
||||
expect(service.vectorInspect).not.toHaveBeenCalled();
|
||||
expect(service.vectorRebuild).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
import Fastify from "fastify";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
import type { CatalogRepository, WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import {
|
||||
readWorkspacePreprocessingStatus,
|
||||
workspacePreprocessingRoutes,
|
||||
type WorkspacePreprocessingRouteDeps,
|
||||
} from "../src/routes/workspace-preprocessing.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
|
||||
const admin: PrincipalContext = {
|
||||
issuer: "test",
|
||||
subject: "admin",
|
||||
roles: ["admin"],
|
||||
permissions: ["session.use", "database.manage"],
|
||||
isAdmin: true,
|
||||
};
|
||||
|
||||
function database(overrides: Partial<WorkspaceDatabase> = {}): WorkspaceDatabase {
|
||||
return {
|
||||
id: "49b6491a-78bb-4cee-b27b-0efaf4419774",
|
||||
workspaceId: "catalog-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "analytics",
|
||||
version: 4,
|
||||
createdAt: "2026-09-05T10:00:00.000Z",
|
||||
updatedAt: "2026-09-05T10:00:00.000Z",
|
||||
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
|
||||
connectionStatus: "reachable",
|
||||
schemaSyncedVersion: 4,
|
||||
metadataContentRevision: 18,
|
||||
preprocessingStatus: "failed",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function routeDeps(
|
||||
current: () => WorkspaceDatabase | undefined,
|
||||
overrides: Partial<WorkspacePreprocessingRouteDeps> = {},
|
||||
): WorkspacePreprocessingRouteDeps {
|
||||
const repository = {
|
||||
getByWorkspace: vi.fn(async () => current()),
|
||||
listSyncRuns: vi.fn(async () => []),
|
||||
getActiveDescriptionGenerationRun: vi.fn(async () => undefined),
|
||||
listSensitivityAnalysisRuns: vi.fn(async () => []),
|
||||
} as unknown as CatalogRepository;
|
||||
const registry = {
|
||||
read: vi.fn(async () => ({
|
||||
workspace: {
|
||||
workspace: {
|
||||
schema_version: 4,
|
||||
id: "catalog-workspace",
|
||||
name: "Catalog workspace",
|
||||
language: "en",
|
||||
},
|
||||
},
|
||||
revision: {
|
||||
id: "catalog-workspace",
|
||||
commit: "a".repeat(40),
|
||||
blob: "b".repeat(40),
|
||||
snapshotPath: "/data/workspaces/catalog-workspace.yaml",
|
||||
},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
return {
|
||||
repository,
|
||||
registry,
|
||||
service: {
|
||||
run: vi.fn(async () => ({ status: "succeeded" })),
|
||||
clear: vi.fn(async () => ({ status: "succeeded" })),
|
||||
} as never,
|
||||
inputFingerprint: {
|
||||
workspaceInputFingerprint: vi.fn(async () => "sha256:current"),
|
||||
} as never,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("reports ready only when the Catalog revision and runtime fingerprint are current", async () => {
|
||||
const ready = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
|
||||
});
|
||||
await expect(readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => ready),
|
||||
)).resolves.toMatchObject({
|
||||
state: "ready",
|
||||
actionable: true,
|
||||
clearable: true,
|
||||
detail: "Catalog revision 18 is indexed.",
|
||||
});
|
||||
|
||||
ready.preprocessingInputFingerprint = "sha256:old";
|
||||
await expect(readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => ready),
|
||||
)).resolves.toMatchObject({
|
||||
state: "required",
|
||||
actionable: true,
|
||||
clearable: true,
|
||||
detail: "Catalog revision 18 is not indexed.",
|
||||
});
|
||||
});
|
||||
|
||||
test("explains a current blocked prerequisite without inventing a run log", async () => {
|
||||
const status = await readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => database({ schemaSyncedVersion: 3 })),
|
||||
);
|
||||
expect(status).toMatchObject({
|
||||
state: "blocked",
|
||||
actionable: false,
|
||||
clearable: true,
|
||||
detail: "Catalog synchronization is required.",
|
||||
reason: "Database configuration v4 is newer than the latest Catalog synchronization v3.",
|
||||
nextStep: "Open Database management and run Synchronize schema.",
|
||||
});
|
||||
expect(status).not.toHaveProperty("lastFailure");
|
||||
});
|
||||
|
||||
test("exposes only the latest sanitized failed-run diagnostic", async () => {
|
||||
const status = await readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(() => database({
|
||||
preprocessingStatus: "failed",
|
||||
preprocessingErrorCode: "schema_index_failed",
|
||||
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
|
||||
})),
|
||||
);
|
||||
expect(status).toMatchObject({
|
||||
state: "failed",
|
||||
actionable: true,
|
||||
clearable: true,
|
||||
reason: expect.stringContaining("schema indexing worker"),
|
||||
lastFailure: {
|
||||
stage: "schema_index",
|
||||
errorCode: "schema_index_failed",
|
||||
finishedAt: "2026-09-05T10:04:00.000Z",
|
||||
},
|
||||
});
|
||||
expect(status).not.toHaveProperty("history");
|
||||
});
|
||||
|
||||
test("reports the durable phase of the active preprocessing run", async () => {
|
||||
const status = await readWorkspacePreprocessingStatus(
|
||||
"catalog-workspace",
|
||||
routeDeps(
|
||||
() => database({ preprocessingStatus: "running" }),
|
||||
{
|
||||
readLatestJob: () => ({
|
||||
status: "active",
|
||||
completedStages: ["catalog_snapshot"],
|
||||
}) as never,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(status).toMatchObject({
|
||||
state: "running",
|
||||
detail: "Building schema vectors and LSH indexes.",
|
||||
progress: { stage: "schema_index", step: 2, totalSteps: 4 },
|
||||
});
|
||||
});
|
||||
|
||||
test("explicitly reruns preprocessing when the current Catalog input is already ready", async () => {
|
||||
const ready = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
});
|
||||
const deps = routeDeps(() => ready);
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async (request) => { request.principal = admin; });
|
||||
workspacePreprocessingRoutes(app, deps);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/workspaces/catalog-workspace/preprocessing",
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
|
||||
expect(deps.service.run).toHaveBeenCalledTimes(1);
|
||||
await app.close();
|
||||
});
|
||||
|
||||
test("runs preprocessing once from the sanctioned admin endpoint and returns the new state", async () => {
|
||||
let current = database();
|
||||
const deps = routeDeps(() => current, {
|
||||
service: {
|
||||
run: vi.fn(async () => {
|
||||
current = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
preprocessingFinishedAt: "2026-09-05T10:04:00.000Z",
|
||||
});
|
||||
return { status: "succeeded" } as never;
|
||||
}),
|
||||
clear: vi.fn(async () => ({ status: "succeeded" } as never)),
|
||||
},
|
||||
});
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async (request) => { request.principal = admin; });
|
||||
workspacePreprocessingRoutes(app, deps);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: "/workspaces/catalog-workspace/preprocessing",
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({ state: "ready", actionable: true });
|
||||
expect(deps.service.run).toHaveBeenCalledTimes(1);
|
||||
await app.close();
|
||||
});
|
||||
|
||||
test("clears only derived preprocessing data from the sanctioned admin endpoint", async () => {
|
||||
let current = database({
|
||||
preprocessingStatus: "succeeded",
|
||||
preprocessedMetadataRevision: 18,
|
||||
preprocessingInputFingerprint: "sha256:current",
|
||||
});
|
||||
const deps = routeDeps(() => current, {
|
||||
service: {
|
||||
run: vi.fn(),
|
||||
clear: vi.fn(async () => {
|
||||
current = database({
|
||||
preprocessingStatus: "failed",
|
||||
preprocessingErrorCode: "derived_data_cleared",
|
||||
preprocessingFinishedAt: "2026-09-05T10:05:00.000Z",
|
||||
});
|
||||
return { status: "succeeded" } as never;
|
||||
}),
|
||||
},
|
||||
});
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async (request) => { request.principal = admin; });
|
||||
workspacePreprocessingRoutes(app, deps);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "DELETE",
|
||||
url: "/workspaces/catalog-workspace/preprocessing",
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({
|
||||
state: "required",
|
||||
clearable: false,
|
||||
detail: "Reference vectors and LSH are empty. Memory is preserved.",
|
||||
});
|
||||
expect(deps.service.clear).toHaveBeenCalledTimes(1);
|
||||
await app.close();
|
||||
});
|
||||
@@ -2,9 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { PreprocessingStateStore } from "../src/workspaces/preprocessing-state.js";
|
||||
import { syncAnnotations } from "../src/workspaces/annotations-sync.js";
|
||||
import { validateWorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
import {
|
||||
WorkspacePreprocessingService,
|
||||
type ChildProcessRequest,
|
||||
@@ -16,73 +14,25 @@ afterEach(() => {
|
||||
roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
|
||||
});
|
||||
|
||||
const semanticRuntime = {
|
||||
internalQdrantUrl: "http://qdrant:6333",
|
||||
internalEmbeddingUrl: "http://embedding:11434",
|
||||
internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
|
||||
internalEmbeddingModel: "qwen3-embedding:0.6b",
|
||||
internalEmbeddingDimensions: 1024,
|
||||
};
|
||||
const workspace = validateWorkspaceDescriptor({
|
||||
workspace: {
|
||||
schema_version: 4,
|
||||
id: "catalog-workspace",
|
||||
name: "Catalog only",
|
||||
language: "en",
|
||||
},
|
||||
});
|
||||
|
||||
const baseWorkspace = parseWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Runtime Lease
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
`);
|
||||
const filesystemWorkspace = parseWorkspaceYaml(`${baseWorkspace ? '' : ''}workspace:
|
||||
schema_version: 4
|
||||
id: fs-workspace
|
||||
name: Filesystem
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: filesystem
|
||||
uri: fs-workspace/evidence
|
||||
`);
|
||||
const privateHttpWorkspace = parseWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: http-workspace
|
||||
name: Http
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: http
|
||||
uris: [http://127.0.0.1/private.md]
|
||||
authentication: none
|
||||
connect_timeout_ms: 1000
|
||||
read_timeout_ms: 2000
|
||||
max_bytes: 100
|
||||
max_redirects: 0
|
||||
allow_private_hosts: true
|
||||
max_cache_bytes: 100
|
||||
`);
|
||||
|
||||
function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id) {
|
||||
function runtime() {
|
||||
return {
|
||||
workspace,
|
||||
workspaceId,
|
||||
workspaceId: "catalog-workspace",
|
||||
workspaceRevision: "a".repeat(40),
|
||||
descriptorBlob: "b".repeat(40),
|
||||
catalogBlob: "c".repeat(40),
|
||||
configLease: {
|
||||
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}-identitysuffix.yaml`,
|
||||
workspaceId,
|
||||
path: `/data/sessions/catalog-workspace/preprocessing/runtime-config/${"a".repeat(40)}.yaml`,
|
||||
workspaceId: "catalog-workspace",
|
||||
workspaceRevision: "a".repeat(40),
|
||||
descriptorBlob: "b".repeat(40),
|
||||
catalogBlob: "c".repeat(40),
|
||||
@@ -90,21 +40,32 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
|
||||
bindingDigest: "sha256:bindings",
|
||||
semanticQdrantUrl: "http://qdrant:6333",
|
||||
effectiveConfig: {
|
||||
schemaVersion: 2,
|
||||
schemaVersion: 3,
|
||||
dwh: {
|
||||
engine: "postgres",
|
||||
database: "analytics",
|
||||
schema: "mart",
|
||||
database: "warehouse",
|
||||
schema: "analytics",
|
||||
transport: "postgres_direct",
|
||||
host: "dwh.internal",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
user: "reader",
|
||||
},
|
||||
vector: { collection: workspaceId, dimensions: 1024, distance: "cosine" },
|
||||
embedding: { id: "ollama/qwen3-embedding:0.6b", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
vector: {
|
||||
collections: {
|
||||
reference: "catalog-workspace-reference",
|
||||
memory: "catalog-workspace-memory",
|
||||
},
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
},
|
||||
embedding: {
|
||||
id: "ollama/qwen3-embedding:0.6b",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
roots: { artifacts: "/data/artifacts", indexes: "/data/indexes" },
|
||||
},
|
||||
effectiveConfigIdentity: "workspace://psd-clinical@v1:" + "d".repeat(64),
|
||||
effectiveConfigIdentity: "workspace://catalog-workspace@v1:" + "d".repeat(64),
|
||||
configFingerprint: "sha256:" + "e".repeat(64),
|
||||
inputFingerprint: "sha256:" + "f".repeat(64),
|
||||
release: () => undefined,
|
||||
@@ -112,385 +73,208 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(workspace = baseWorkspace) {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
|
||||
roots.push(dataRoot);
|
||||
const requests: ChildProcessRequest[] = [];
|
||||
const runChild = vi.fn(async (request: ChildProcessRequest) => {
|
||||
requests.push(request);
|
||||
return { exitCode: 0, stdout: JSON.stringify({ status: "succeeded" }), stderr: "" };
|
||||
});
|
||||
const service = new WorkspacePreprocessingService({
|
||||
dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(workspace),
|
||||
runChild,
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: true }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
});
|
||||
return { dataRoot, runChild, requests, service };
|
||||
const database = {
|
||||
id: "database-1",
|
||||
workspaceId: "catalog-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "analytics",
|
||||
version: 4,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
binding: { transport: "postgres_direct", host: "db", port: 5432, username: "reader" },
|
||||
connectionStatus: "reachable",
|
||||
schemaSyncedVersion: 4,
|
||||
metadataContentRevision: 9,
|
||||
preprocessingStatus: "running",
|
||||
} as const;
|
||||
|
||||
function repository(overrides: Record<string, unknown> = {}) {
|
||||
const finishPreprocessing = vi.fn(async () => ({
|
||||
...database,
|
||||
preprocessingStatus: "succeeded" as const,
|
||||
preprocessedMetadataRevision: 9,
|
||||
}));
|
||||
return {
|
||||
beginPreprocessing: vi.fn(async () => ({ kind: "started" as const, database })),
|
||||
finishPreprocessing,
|
||||
clearPreprocessing: vi.fn(async () => ({ kind: "cleared" as const, database })),
|
||||
getByWorkspace: vi.fn(async () => database),
|
||||
listTables: vi.fn(async () => [{
|
||||
id: "table-1", databaseId: database.id, name: "orders",
|
||||
description: "Curated orders", generatedDescription: "Generated orders",
|
||||
sourceComment: "PostgreSQL orders", version: 1,
|
||||
createdAt: database.createdAt, updatedAt: database.updatedAt,
|
||||
lastSyncedDatabaseVersion: 4, lastSyncedAt: database.updatedAt,
|
||||
}]),
|
||||
listColumns: vi.fn(async () => [{
|
||||
id: "column-1", tableId: "table-1", name: "customer_id", ordinalPosition: 1,
|
||||
dataType: "uuid", isNullable: false, defaultExpression: null,
|
||||
primaryKeyPosition: null, isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1,
|
||||
sourceComment: "PostgreSQL customer", description: null,
|
||||
generatedDescription: "Generated customer", sensitive: true,
|
||||
sensitivityReason: "identifier", lastSyncedDatabaseVersion: 4,
|
||||
lastSyncedAt: database.updatedAt, version: 1,
|
||||
createdAt: database.createdAt, updatedAt: database.updatedAt,
|
||||
}]),
|
||||
listRelationships: vi.fn(async () => []),
|
||||
listLogicalRelationships: vi.fn(async () => []),
|
||||
...overrides,
|
||||
} as any;
|
||||
}
|
||||
|
||||
test("preprocess dwh uses fixed argv and resumes outer state without rerunning a completed stage", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const first = await f.service.preprocessDwh({ workspaceId: "psd-clinical" });
|
||||
expect(first).toMatchObject({
|
||||
status: "succeeded",
|
||||
code: "ok",
|
||||
operation: "preprocess dwh",
|
||||
completedStages: ["dwh"],
|
||||
childRuns: { dwh: "d".repeat(32) },
|
||||
});
|
||||
expect((f.runChild.mock.calls[0]![0] as ChildProcessRequest).argv).toEqual([
|
||||
"preprocess", "dwh", "--steps", "introspect,lsh", "--json", "-c", "/dev/fd/3",
|
||||
]);
|
||||
|
||||
const second = await f.service.preprocessDwh({
|
||||
workspaceId: "psd-clinical",
|
||||
resumeRunId: first.runId!,
|
||||
});
|
||||
expect(second.status).toBe("unchanged");
|
||||
expect(f.runChild).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("schema suggest-fks publishes a candidate artifact and blocks full runs for manual review", async () => {
|
||||
const f = fixture();
|
||||
f.runChild
|
||||
.mockResolvedValueOnce({
|
||||
function service(options: {
|
||||
repository?: any;
|
||||
runChild?: (request: ChildProcessRequest) => Promise<{
|
||||
exitCode: number; stdout: string; stderr: string;
|
||||
}>;
|
||||
semanticPreflight?: () => Promise<
|
||||
{ ok: true } | { ok: false; code: "semantic_index_incompatible" }
|
||||
>;
|
||||
} = {}) {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), "tht-catalog-preprocessing-"));
|
||||
roots.push(dataRoot);
|
||||
return new WorkspacePreprocessingService({
|
||||
dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(),
|
||||
runChild: options.runChild ?? (async () => ({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
|
||||
stderr: "",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: []\n",
|
||||
}),
|
||||
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
|
||||
stderr: "",
|
||||
})),
|
||||
semanticPreflight: options.semanticPreflight ?? (async () => ({ ok: true })),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
catalogRepository: options.repository,
|
||||
});
|
||||
}
|
||||
|
||||
test("complete preprocessing snapshots Catalog metadata and commits its PostgreSQL state", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async (request: ChildProcessRequest) => {
|
||||
const snapshotPath = request.argv[request.argv.indexOf("--catalog-metadata") + 1]!;
|
||||
const snapshot = JSON.parse(readFileSync(snapshotPath, "utf8"));
|
||||
expect(snapshot).toMatchObject({
|
||||
workspaceId: "catalog-workspace",
|
||||
databaseName: "warehouse",
|
||||
metadataContentRevision: 9,
|
||||
tables: [{
|
||||
name: "orders",
|
||||
description: "Curated orders",
|
||||
descriptionSource: "curated",
|
||||
columns: [{
|
||||
name: "customer_id",
|
||||
description: "Generated customer",
|
||||
descriptionSource: "generated",
|
||||
sensitive: true,
|
||||
}],
|
||||
}],
|
||||
});
|
||||
|
||||
const result = await f.service.run({ workspaceId: "psd-clinical" });
|
||||
expect(result).toMatchObject({
|
||||
status: "blocked",
|
||||
code: "manual_review_required",
|
||||
completedStages: ["dwh", "fk_suggest"],
|
||||
});
|
||||
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv[0])).toEqual(["preprocess", "schema"]);
|
||||
});
|
||||
|
||||
test("schema check requires the exact candidate digest and stages annotations via a temp file without recording a review", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: []\n",
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
|
||||
await expect(f.service.checkSchema({
|
||||
workspaceId: "psd-clinical",
|
||||
annotationsYaml: "tables: {}\n",
|
||||
reviewedCandidatesDigest: "sha256:" + "f".repeat(64),
|
||||
})).resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
|
||||
const reviewedDigest = suggest.artifactIdentities![0]!.digest;
|
||||
let stagedPath = "";
|
||||
f.runChild.mockImplementationOnce(async (request: ChildProcessRequest) => {
|
||||
stagedPath = request.argv[request.argv.indexOf("--annotations") + 1]!;
|
||||
expect(readFileSync(stagedPath, "utf8")).toBe("tables: {}\n");
|
||||
expect(request.argv).toEqual([
|
||||
"schema", "check", "--annotations", stagedPath,
|
||||
"--reviewed-candidates", reviewedDigest,
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
]);
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
orphan_count: 0,
|
||||
annotations_digest: "sha256:annotations",
|
||||
reviewed_candidates_digest: reviewedDigest,
|
||||
}),
|
||||
stdout: JSON.stringify({ counts: { tables: 1, columns: 1, relationships: 0 } }),
|
||||
stderr: "",
|
||||
};
|
||||
});
|
||||
|
||||
const checked = await f.service.checkSchema({
|
||||
workspaceId: "psd-clinical",
|
||||
annotationsYaml: "tables: {}\n",
|
||||
reviewedCandidatesDigest: reviewedDigest,
|
||||
});
|
||||
expect(checked).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
expect(() => readFileSync(stagedPath, "utf8")).toThrow();
|
||||
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
|
||||
expect(state.readFkReview(suggest.runId!)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("index schema fails closed when semantic preflight refuses the collection", async () => {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
|
||||
roots.push(dataRoot);
|
||||
const runChild = vi.fn();
|
||||
const service = new WorkspacePreprocessingService({
|
||||
dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(baseWorkspace),
|
||||
runChild,
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
const result = await service({ repository: catalog, runChild }).run({
|
||||
workspaceId: "catalog-workspace",
|
||||
});
|
||||
|
||||
const result = await service.indexSchema({ workspaceId: "psd-clinical" });
|
||||
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
|
||||
expect(runChild).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("filesystem Evidence proceeds after materialization and private HTTP hosts outside the allowlist are refused", async () => {
|
||||
const filesystem = fixture(filesystemWorkspace);
|
||||
filesystem.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1 } }),
|
||||
stderr: "",
|
||||
});
|
||||
const materialized = await filesystem.service.preprocessEvidence({ workspaceId: "fs-workspace" });
|
||||
expect(materialized).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
expect(filesystem.runChild).toHaveBeenCalledTimes(1);
|
||||
|
||||
const httpDataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-"));
|
||||
roots.push(httpDataRoot);
|
||||
const httpService = new WorkspacePreprocessingService({
|
||||
dataRoot: httpDataRoot,
|
||||
acquireActiveRuntime: async () => runtime(privateHttpWorkspace, "http-workspace"),
|
||||
runChild: vi.fn(),
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: true }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
httpPrivateHostAllowlist: ["metadata.internal"],
|
||||
});
|
||||
|
||||
const refused = await httpService.preprocessEvidence({ workspaceId: "http-workspace" });
|
||||
expect(refused).toMatchObject({ status: "failed", code: "egress_policy_refused" });
|
||||
});
|
||||
|
||||
test("full runs follow the explicit order and finish unchanged when no Evidence source exists", async () => {
|
||||
const f = fixture();
|
||||
f.runChild
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }),
|
||||
stderr: "",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 0,
|
||||
candidate_digest: "sha256:" + "0".repeat(64),
|
||||
candidate_yaml: "tables: []\n",
|
||||
}),
|
||||
stderr: "",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 },
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const result = await f.service.run({ workspaceId: "psd-clinical" });
|
||||
expect(result).toMatchObject({
|
||||
status: "succeeded",
|
||||
code: "ok",
|
||||
completedStages: ["dwh", "fk_suggest", "schema_index"],
|
||||
warnings: ["workspace has no Evidence source"],
|
||||
completedStages: ["catalog_snapshot", "catalog_metadata", "lsh", "schema_index"],
|
||||
});
|
||||
expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv.slice(0, 2).join(" "))).toEqual([
|
||||
"preprocess dwh",
|
||||
"schema suggest-fks",
|
||||
"vector index-schema",
|
||||
]);
|
||||
expect(runChild).toHaveBeenCalledWith(expect.objectContaining({
|
||||
argv: [
|
||||
"preprocess", "catalog", "--catalog-metadata",
|
||||
expect.stringMatching(/\/catalog-metadata\.json$/),
|
||||
"--json", "-c", "/dev/fd/3",
|
||||
],
|
||||
}));
|
||||
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
|
||||
"catalog-workspace",
|
||||
9,
|
||||
"sha256:" + "f".repeat(64),
|
||||
{ status: "succeeded" },
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("schema accept validates the synced Git blob and records the review", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: {}\n",
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
|
||||
const runId = suggest.runId!;
|
||||
const candidateDigest = suggest.artifactIdentities![0]!.digest;
|
||||
const synced = syncAnnotations({
|
||||
dataRoot: f.dataRoot,
|
||||
workspaceId: "psd-clinical",
|
||||
commit: "a".repeat(40),
|
||||
blobId: "b".repeat(40),
|
||||
contents: Buffer.from("tables: {}\n"),
|
||||
});
|
||||
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
orphan_count: 0,
|
||||
annotations_digest: synced.contentDigest,
|
||||
reviewed_candidates_digest: candidateDigest,
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const result = await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
|
||||
expect(result).toMatchObject({ status: "succeeded", code: "ok", operation: "schema accept" });
|
||||
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
|
||||
expect(state.readFkReview(runId)).toMatchObject({
|
||||
reviewedCandidatesDigest: candidateDigest,
|
||||
annotationsDigest: synced.contentDigest,
|
||||
workspaceRevision: "a".repeat(40),
|
||||
blobId: "b".repeat(40),
|
||||
});
|
||||
test("preprocessing fails closed when the PostgreSQL Catalog is unavailable", async () => {
|
||||
const result = await service().run({ workspaceId: "catalog-workspace" });
|
||||
expect(result).toMatchObject({ status: "failed", code: "catalog_not_ready" });
|
||||
});
|
||||
|
||||
test("schema accept fails closed without --yes, for an unknown run, or with no synced annotations", async () => {
|
||||
const f = fixture();
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({
|
||||
status: "succeeded",
|
||||
candidate_count: 1,
|
||||
candidate_digest: "sha256:" + "e".repeat(64),
|
||||
candidate_yaml: "tables: {}\n",
|
||||
}),
|
||||
stderr: "",
|
||||
test("preprocessing refuses a concurrent Catalog run without touching derived data", async () => {
|
||||
const catalog = repository({
|
||||
beginPreprocessing: vi.fn(async () => ({ kind: "already_running" as const })),
|
||||
});
|
||||
const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" });
|
||||
const runId = suggest.runId!;
|
||||
const runChild = vi.fn();
|
||||
|
||||
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: false }))
|
||||
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
const result = await service({ repository: catalog, runChild }).run({
|
||||
workspaceId: "catalog-workspace",
|
||||
});
|
||||
|
||||
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId: "e".repeat(32), yes: true }))
|
||||
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
|
||||
await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true }))
|
||||
.resolves.toMatchObject({ status: "failed", code: "annotation_invalid" });
|
||||
expect(f.runChild).toHaveBeenCalledTimes(1);
|
||||
expect(result).toMatchObject({ status: "failed", code: "preprocessing_conflict" });
|
||||
expect(runChild).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("full runs continue after schema accept only when the accepted blob matches the current revision", async () => {
|
||||
const f = fixture();
|
||||
const revision = "a".repeat(40);
|
||||
f.runChild
|
||||
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const blocked = await f.service.run({ workspaceId: "psd-clinical" });
|
||||
expect(blocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
|
||||
const runId = blocked.runId!;
|
||||
const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" });
|
||||
const candidateDigest = state.readFkCandidates(runId)!.digest;
|
||||
|
||||
const synced = syncAnnotations({
|
||||
dataRoot: f.dataRoot,
|
||||
workspaceId: "psd-clinical",
|
||||
commit: revision,
|
||||
blobId: "b".repeat(40),
|
||||
contents: Buffer.from("tables: {}\n"),
|
||||
test("preprocessing records a failed PostgreSQL state when its hidden worker fails", async () => {
|
||||
const catalog = repository();
|
||||
const instance = service({
|
||||
repository: catalog,
|
||||
runChild: async () => ({ exitCode: 1, stdout: "", stderr: "private failure" }),
|
||||
});
|
||||
|
||||
// Accept writes the review; the resume then passes the gate and reaches schema indexing.
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", orphan_count: 0, annotations_digest: synced.contentDigest, reviewed_candidates_digest: candidateDigest }),
|
||||
stderr: "",
|
||||
});
|
||||
await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true });
|
||||
|
||||
f.runChild.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 } }),
|
||||
stderr: "",
|
||||
});
|
||||
const resumed = await f.service.run({ workspaceId: "psd-clinical", resumeRunId: runId });
|
||||
expect(resumed).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
|
||||
// A review whose accepted blob digest no longer matches the current revision stays blocked.
|
||||
const second = fixture();
|
||||
second.runChild
|
||||
.mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" })
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }),
|
||||
stderr: "",
|
||||
});
|
||||
const secondBlocked = await second.service.run({ workspaceId: "psd-clinical" });
|
||||
const secondRunId = secondBlocked.runId!;
|
||||
const secondState = new PreprocessingStateStore({ dataRoot: second.dataRoot, workspaceId: "psd-clinical" });
|
||||
const secondCandidate = secondState.readFkCandidates(secondRunId)!.digest;
|
||||
secondState.writeFkReview(secondRunId, {
|
||||
reviewedCandidatesDigest: secondCandidate,
|
||||
annotationsDigest: "sha256:" + "0".repeat(64),
|
||||
workspaceRevision: revision,
|
||||
blobId: "b".repeat(40),
|
||||
});
|
||||
const stillBlocked = await second.service.run({ workspaceId: "psd-clinical", resumeRunId: secondRunId });
|
||||
expect(stillBlocked).toMatchObject({ status: "blocked", code: "manual_review_required" });
|
||||
await expect(instance.run({ workspaceId: "catalog-workspace" })).rejects.toThrow(
|
||||
"workspace child failed",
|
||||
);
|
||||
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
|
||||
"catalog-workspace",
|
||||
9,
|
||||
"sha256:" + "f".repeat(64),
|
||||
{ status: "failed", errorCode: "schema_index_failed" },
|
||||
);
|
||||
});
|
||||
|
||||
test("vector rebuild recreates the full collection contract including keyword indexes", async () => {
|
||||
const f = fixture();
|
||||
// mock fetch: DELETE ok, then reconcileCollection self-heals create + indexes (real fetch in deps)
|
||||
const calls: string[] = [];
|
||||
const fakeFetch = async (url: string, init?: any) => {
|
||||
calls.push(`${init?.method ?? "GET"} ${url}`);
|
||||
if ((init?.method ?? "GET") === "DELETE") return new Response("", { status: 200 });
|
||||
if (url.endsWith("/collections/psd-clinical") && init?.method === "PUT") return new Response("", { status: 200 });
|
||||
if (url.endsWith("/collections/psd-clinical") && init?.method === "GET") {
|
||||
return new Response(JSON.stringify({ result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, payload_schema: { content_hash: { data_type: "keyword" }, document_id: { data_type: "keyword" }, kind: { data_type: "keyword" }, record_key: { data_type: "keyword" }, record_kind: { data_type: "keyword" }, vector_generation: { data_type: "keyword" }, workspace_id: { data_type: "keyword" }, workspace_revision: { data_type: "keyword" } } } }), { status: 200 });
|
||||
}
|
||||
if (url.endsWith("/collections/psd-clinical/index") && init?.method === "PUT") return new Response("", { status: 200 });
|
||||
return new Response(JSON.stringify({ result: {} }), { status: 200 });
|
||||
};
|
||||
const service = new WorkspacePreprocessingService({
|
||||
dataRoot: f.dataRoot,
|
||||
acquireActiveRuntime: async () => runtime(baseWorkspace),
|
||||
runChild: vi.fn(),
|
||||
listSessions: async () => [],
|
||||
semanticPreflight: async () => ({ ok: true }),
|
||||
evidencePreflight: async () => ({ ok: true }),
|
||||
test("semantic preflight failure is persisted before returning", async () => {
|
||||
const catalog = repository();
|
||||
const result = await service({
|
||||
repository: catalog,
|
||||
semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }),
|
||||
}).run({ workspaceId: "catalog-workspace" });
|
||||
|
||||
expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" });
|
||||
expect(catalog.finishPreprocessing).toHaveBeenCalledWith(
|
||||
"catalog-workspace",
|
||||
9,
|
||||
"sha256:" + "f".repeat(64),
|
||||
{ status: "failed", errorCode: "semantic_index_incompatible" },
|
||||
);
|
||||
});
|
||||
|
||||
test("clear invalidates Catalog readiness before clearing only derived worker data", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ counts: { referenceCollections: 1, derivedPaths: 3 } }),
|
||||
stderr: "",
|
||||
}));
|
||||
|
||||
const result = await service({ repository: catalog, runChild }).clear({
|
||||
workspaceId: "catalog-workspace",
|
||||
});
|
||||
// replace global fetch used by vectorRebuild/reconcileCollection
|
||||
const original = globalThis.fetch;
|
||||
globalThis.fetch = fakeFetch as any;
|
||||
try {
|
||||
const result = await service.vectorRebuild({ workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true });
|
||||
expect(result).toMatchObject({ status: "succeeded", code: "ok" });
|
||||
} finally {
|
||||
globalThis.fetch = original;
|
||||
}
|
||||
expect(calls.some((c) => c.startsWith("DELETE "))).toBe(true);
|
||||
|
||||
expect(catalog.clearPreprocessing).toHaveBeenCalledWith("catalog-workspace");
|
||||
expect(runChild).toHaveBeenCalledWith({
|
||||
argv: ["preprocess", "clear", "--json", "-c", "/dev/fd/3"],
|
||||
configPath: expect.any(String),
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
status: "succeeded",
|
||||
code: "ok",
|
||||
operation: "preprocess clear",
|
||||
counts: { referenceCollections: 1, derivedPaths: 3 },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -47,6 +47,15 @@ test("job state creates durable 0600 JSON and enforces same-revision resume", as
|
||||
embeddingId: "ollama/qwen3-embedding:0.6b",
|
||||
embeddingDimensions: 1024,
|
||||
});
|
||||
expect(store.readLatestJob()).toMatchObject({
|
||||
runId: job.runId,
|
||||
status: "active",
|
||||
completedStages: [],
|
||||
});
|
||||
|
||||
job.completedStages.push("catalog_snapshot");
|
||||
store.writeJob(job);
|
||||
expect(store.readLatestJob()?.completedStages).toEqual(["catalog_snapshot"]);
|
||||
|
||||
await expect(store.beginJob({
|
||||
operation: "preprocess dwh",
|
||||
|
||||
@@ -19,11 +19,6 @@ const validYaml = `workspace:
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
|
||||
@@ -34,119 +29,12 @@ function withFilesystemEvidence(source: string, id = "psd-clinical"): string {
|
||||
`);
|
||||
}
|
||||
|
||||
function withDwhRestTransport(source: string): string {
|
||||
return source.replace(
|
||||
"supported_transports: [postgres_direct]",
|
||||
"supported_transports: [postgres_direct, rest_api]",
|
||||
);
|
||||
}
|
||||
|
||||
function withDwhRestDiagnostic(source: string): string {
|
||||
return withDwhRestTransport(source).concat(`diagnostics:
|
||||
dwh_rest:
|
||||
method: GET
|
||||
path: /health
|
||||
auth: none
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
`);
|
||||
}
|
||||
|
||||
function withEmbeddingDiagnostic(source: string): string {
|
||||
return source.concat(`diagnostics:
|
||||
embedding:
|
||||
method: GET
|
||||
path: /models
|
||||
auth: none
|
||||
response:
|
||||
model: model
|
||||
dimensions: dimensions
|
||||
`);
|
||||
}
|
||||
|
||||
function withDwhRestAndEmbeddingDiagnostics(source: string): string {
|
||||
return withDwhRestTransport(source).concat(`diagnostics:
|
||||
dwh_rest:
|
||||
method: GET
|
||||
path: /health
|
||||
auth: none
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
embedding:
|
||||
method: GET
|
||||
path: /models
|
||||
auth: none
|
||||
response:
|
||||
model: model
|
||||
dimensions: dimensions
|
||||
`);
|
||||
}
|
||||
|
||||
function withVectorRestTransport(source: string): string {
|
||||
return source.replace(
|
||||
"supported_transports: [pgvector_direct]",
|
||||
"supported_transports: [pgvector_direct, rest_api]",
|
||||
);
|
||||
}
|
||||
|
||||
function withVectorMetadataDiagnostic(source: string): string {
|
||||
return withVectorRestTransport(source).concat(`diagnostics:
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: none
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
`);
|
||||
}
|
||||
|
||||
function withReversibleVectorProbe(source: string): string {
|
||||
return withVectorRestTransport(source).concat(`diagnostics:
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: none
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
reversible_probe:
|
||||
method: POST
|
||||
path: /probe
|
||||
auth: bearer
|
||||
response:
|
||||
operation: operation
|
||||
`);
|
||||
}
|
||||
|
||||
function legacyV1Yaml(source = validYaml): string {
|
||||
return source
|
||||
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
|
||||
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace(" provider: ollama_internal", " provider: ollama_compatible")
|
||||
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
|
||||
.replace("schema_version: 4", "schema_version: 1");
|
||||
return source.replace("schema_version: 4", "schema_version: 1");
|
||||
}
|
||||
|
||||
function legacyV2Yaml(source = validYaml): string {
|
||||
return source
|
||||
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
|
||||
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace(" provider: ollama_internal", " provider: ollama_compatible")
|
||||
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
|
||||
.replace(" dimensions: 1024", " dimensions: 768")
|
||||
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
|
||||
.replace("schema_version: 4", "schema_version: 2");
|
||||
return source.replace("schema_version: 4", "schema_version: 2");
|
||||
}
|
||||
|
||||
const runFile = promisify(execFile);
|
||||
@@ -605,23 +493,6 @@ test("keeps content-only historical descriptor revisions distinguishable by comm
|
||||
expect(oldPinned.workspace).toEqual(newPinned.workspace);
|
||||
});
|
||||
|
||||
test.each([
|
||||
["adds", validYaml, withDwhRestDiagnostic(validYaml)],
|
||||
["removes", withDwhRestDiagnostic(validYaml), validYaml],
|
||||
])("pulls a curator change that %s a diagnostics branch without API rewrite", async (_operation, baseSource, remoteSource) => {
|
||||
const remote = await fixture(baseSource);
|
||||
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
||||
await registry.bootstrap();
|
||||
const initial = await registry.read("psd-clinical");
|
||||
writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), remoteSource);
|
||||
await git(remote.source, ["add", "psd-clinical/workspace.yaml"]);
|
||||
await git(remote.source, ["commit", "-m", `Registry ${_operation} diagnostic branch`]);
|
||||
await git(remote.source, ["push", "origin", "main"]);
|
||||
|
||||
await registry.pull();
|
||||
const updated = await registry.read("psd-clinical");
|
||||
expect(updated.revision.commit).not.toBe(initial.revision.commit);
|
||||
});
|
||||
test.each([
|
||||
["v1", legacyV1Yaml()],
|
||||
["v2", legacyV2Yaml()],
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
mkdirSync,
|
||||
@@ -27,6 +26,7 @@ import {
|
||||
renderActiveWorkspaceRuntime,
|
||||
renderWorkspaceRuntimeFromSnapshotPath,
|
||||
} from "../src/workspaces/runtime-config-lease.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const runFile = promisify(execFile);
|
||||
const roots: string[] = [];
|
||||
@@ -70,11 +70,6 @@ workspaces: [{id: psd-clinical, name: Runtime Lease}]
|
||||
id: psd-clinical
|
||||
name: Runtime Lease
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
evidence:
|
||||
source:
|
||||
type: filesystem
|
||||
@@ -88,9 +83,6 @@ evidence:
|
||||
await git(source, ["push", "origin", "main"]);
|
||||
|
||||
mkdirSync(secretRoot);
|
||||
const passwordFile = join(secretRoot, "dwh-password");
|
||||
writeFileSync(passwordFile, "secret", { mode: 0o600 });
|
||||
chmodSync(passwordFile, 0o600);
|
||||
mkdirSync(dataRoot);
|
||||
|
||||
const registryConfig: WorkspaceRegistryConfig = {
|
||||
@@ -107,12 +99,31 @@ evidence:
|
||||
const registry = new WorkspaceRegistry(registryConfig);
|
||||
await registry.bootstrap();
|
||||
const revision = (await registry.list())[0];
|
||||
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse.internal");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "reader");
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE", passwordFile);
|
||||
const workspaceSecretStore = new WorkspaceSecretStore({
|
||||
root: join(root, "vault"),
|
||||
runtimeRoot: join(root, "runtime-secrets"),
|
||||
installationId: "test",
|
||||
});
|
||||
workspaceSecretStore.putMany("psd-clinical", { "catalog.dwh.password": "secret" });
|
||||
const catalogDatabase = {
|
||||
id: "database-1",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres" as const,
|
||||
databaseName: "analytics",
|
||||
schema: "mart",
|
||||
binding: {
|
||||
transport: "postgres_direct" as const,
|
||||
host: "warehouse.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
version: 1,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
connectionStatus: "reachable" as const,
|
||||
metadataContentRevision: 1,
|
||||
preprocessingStatus: "failed" as const,
|
||||
};
|
||||
|
||||
return {
|
||||
dataRoot,
|
||||
@@ -121,6 +132,8 @@ evidence:
|
||||
registry,
|
||||
registryConfig,
|
||||
revision,
|
||||
workspaceSecretStore,
|
||||
catalogDatabase,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -140,6 +153,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const active = await renderActiveWorkspaceRuntime({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -150,6 +165,8 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
expect(active.renderedConfig).toBe(direct.renderedConfig);
|
||||
@@ -158,27 +175,6 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
||||
expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
test("renders a revision-qualified annotations root for the active revision", async () => {
|
||||
const f = await fixture();
|
||||
const active = await renderActiveWorkspaceRuntime({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
registryConfig: f.registryConfig,
|
||||
harnessDir: f.harnessDir,
|
||||
configPath: "config/tht.yaml",
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
});
|
||||
const rendered = parse(active.renderedConfig) as Record<string, any>;
|
||||
|
||||
expect(rendered.paths.annotations_root).toBe(
|
||||
join(f.dataRoot, "sessions", "psd-clinical", "revisions", f.revision.commit, "artifacts"),
|
||||
);
|
||||
expect(rendered.roots.annotations_root).toBe(rendered.paths.annotations_root);
|
||||
expect(rendered.paths.artifacts).toBe(join(f.dataRoot, "sessions", "psd-clinical", "artifacts"));
|
||||
});
|
||||
|
||||
test("deterministic operator leases are keyed by logical identity and stable across calls", async () => {
|
||||
const f = await fixture();
|
||||
const first = await publishDeterministicRuntimeConfigLease({
|
||||
@@ -190,6 +186,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const second = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -200,6 +198,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
const suffix = first.inputFingerprint.slice(7, 23);
|
||||
@@ -236,7 +236,11 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
path: first.path,
|
||||
});
|
||||
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal");
|
||||
const changedDatabase = {
|
||||
...f.catalogDatabase,
|
||||
binding: { ...f.catalogDatabase.binding, host: "warehouse-two.internal" },
|
||||
version: 2,
|
||||
};
|
||||
const changed = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
@@ -246,6 +250,8 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: changedDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
expect(changed.path).not.toBe(first.path);
|
||||
expect(changed.inputFingerprint).not.toBe(first.inputFingerprint);
|
||||
@@ -267,6 +273,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
})).toThrow(/trusted runtime snapshot/i);
|
||||
expect(() => renderWorkspaceRuntimeFromSnapshotPath({
|
||||
snapshotPath: symlink,
|
||||
@@ -275,6 +283,8 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
})).toThrow(/trusted runtime snapshot/i);
|
||||
});
|
||||
|
||||
@@ -288,6 +298,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const lease = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -298,6 +310,8 @@ test("operator lease and session snapshot produce byte-identical effective DWH b
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
const sessionCanonical = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(parse(session.renderedConfig)));
|
||||
@@ -319,6 +333,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
const firstIdentity = firstLease.effectiveConfigIdentity;
|
||||
|
||||
@@ -341,6 +357,8 @@ test("a content-only Evidence commit keeps the same effective config identity wi
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
catalogDatabase: f.catalogDatabase,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
expect(secondLease.workspaceRevision).toBe(current.commit);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync,
|
||||
chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -25,11 +25,6 @@ const canonicalWorkspace = `workspace:
|
||||
id: psd-clinical
|
||||
name: Runtime handoff
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
const filesystemWorkspace = `${canonicalWorkspace}evidence:
|
||||
@@ -106,6 +101,38 @@ async function fixture(workspaceSource = filesystemWorkspace) {
|
||||
const registry = new WorkspaceRegistry(registryConfig);
|
||||
await registry.bootstrap();
|
||||
const revision = (await registry.list())[0];
|
||||
const workspaceSecretStore = new WorkspaceSecretStore({
|
||||
root: join(root, "workspace-secrets"),
|
||||
runtimeRoot: join(root, "workspace-secret-runtime"),
|
||||
installationId: "test",
|
||||
});
|
||||
workspaceSecretStore.putMany("psd-clinical", {
|
||||
"catalog.dwh.password": "dwh-password-value",
|
||||
"evidence.signed_urls": secretContents["evidence-signed-urls.json"],
|
||||
"evidence.access_key": secretContents["evidence-access"],
|
||||
"evidence.secret_key": secretContents["evidence-secret"],
|
||||
"evidence.session_token": secretContents["evidence-token"],
|
||||
});
|
||||
const catalogDatabase = {
|
||||
id: "database-1",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres" as const,
|
||||
databaseName: "analytics",
|
||||
schema: "mart",
|
||||
binding: {
|
||||
transport: "postgres_direct" as const,
|
||||
host: "dwh.invalid",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
version: 1,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
connectionStatus: "reachable" as const,
|
||||
metadataContentRevision: 1,
|
||||
preprocessingStatus: "failed" as const,
|
||||
};
|
||||
const catalogRepository = { getByWorkspace: async () => catalogDatabase } as any;
|
||||
const environment = {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
|
||||
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.invalid",
|
||||
@@ -119,7 +146,10 @@ async function fixture(workspaceSource = filesystemWorkspace) {
|
||||
};
|
||||
for (const [name, value] of Object.entries(environment)) vi.stubEnv(name, value);
|
||||
vi.stubEnv("THT_HOME", join(root, "home"));
|
||||
return { root, source, dataRoot, secretRoot, registry, registryConfig, revision };
|
||||
return {
|
||||
root, source, dataRoot, secretRoot, registry, registryConfig, revision,
|
||||
workspaceSecretStore, catalogDatabase, catalogRepository,
|
||||
};
|
||||
}
|
||||
|
||||
function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
|
||||
@@ -130,6 +160,8 @@ function runnerFor(f: Awaited<ReturnType<typeof fixture>>): ThtRunner {
|
||||
dataRoot: f.dataRoot,
|
||||
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
catalogRepository: f.catalogRepository,
|
||||
} as any);
|
||||
}
|
||||
|
||||
@@ -165,7 +197,7 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
|
||||
runtimeRoot,
|
||||
installationId: "test",
|
||||
});
|
||||
secretStore.put("psd-clinical", "dwh.password", "vault-runtime-password");
|
||||
secretStore.put("psd-clinical", "catalog.dwh.password", "vault-runtime-password");
|
||||
const runner = new ThtRunner({
|
||||
thtBin,
|
||||
harnessDir,
|
||||
@@ -174,9 +206,10 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
|
||||
runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"),
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
workspaceSecretStore: secretStore,
|
||||
catalogRepository: f.catalogRepository,
|
||||
} as any);
|
||||
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const rendered = parse(readFileSync(lease.path, "utf8")) as {
|
||||
database: { password_file: string };
|
||||
};
|
||||
@@ -185,30 +218,11 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
|
||||
expect(existsSync(rendered.database.password_file)).toBe(false);
|
||||
});
|
||||
|
||||
test("ThtRunner binds and cleans the effective relationship snapshot with its runtime lease", async () => {
|
||||
const f = await fixture();
|
||||
const runner = runnerFor(f);
|
||||
const relationships = JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
workspaceId: "psd-clinical",
|
||||
relationships: [],
|
||||
});
|
||||
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath, relationships);
|
||||
const rendered = parse(readFileSync(lease.path, "utf8")) as {
|
||||
paths: { effective_relationships: string };
|
||||
};
|
||||
|
||||
expect(readFileSync(rendered.paths.effective_relationships, "utf8")).toBe(relationships);
|
||||
lease.release();
|
||||
expect(existsSync(rendered.paths.effective_relationships)).toBe(false);
|
||||
});
|
||||
|
||||
test("separate runtime leases hand off byte-identical revision Evidence configs accepted by tht", async () => {
|
||||
const f = await fixture();
|
||||
const runner = runnerFor(f);
|
||||
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const second = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const second = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const expectedRoot = join(
|
||||
f.registryConfig.root,
|
||||
"snapshots",
|
||||
@@ -263,7 +277,7 @@ test("separate runtime leases hand off byte-identical revision Evidence configs
|
||||
test("real Evidence-content-only commit changes runtime identity and root with identical descriptor YAML", async () => {
|
||||
const f = await fixture();
|
||||
const runner = runnerFor(f);
|
||||
const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const first = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const descriptorBefore = readFileSync(f.revision.snapshotPath, "utf8");
|
||||
writeFileSync(
|
||||
join(f.source, "psd-clinical", "evidence", "guide.md"),
|
||||
@@ -274,7 +288,7 @@ test("real Evidence-content-only commit changes runtime identity and root with i
|
||||
await git(f.source, ["push", "origin", "main"]);
|
||||
await f.registry.pull();
|
||||
const current = (await f.registry.list())[0];
|
||||
const second = runner.acquireWorkspaceRuntime(current.snapshotPath);
|
||||
const second = await runner.acquireWorkspaceRuntime(current.snapshotPath);
|
||||
|
||||
try {
|
||||
expect(current.commit).not.toBe(f.revision.commit);
|
||||
@@ -317,13 +331,13 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
|
||||
max_cache_bytes: 67890
|
||||
`));
|
||||
const runner = runnerFor(f);
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
try {
|
||||
const yaml = readFileSync(lease.path, "utf8");
|
||||
expect(parse(yaml).evidence.sources).toEqual([{
|
||||
type: "http",
|
||||
provenance_urls: ["https://evidence.example.test/guide.md"],
|
||||
signed_urls_file: realpathSync(join(f.secretRoot, "evidence-signed-urls.json")),
|
||||
signed_urls_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
connect_timeout: 1.25,
|
||||
read_timeout: 30.001,
|
||||
max_bytes: 12_345,
|
||||
@@ -343,7 +357,7 @@ test("signed HTTP Evidence resolves its file binding and config check never capt
|
||||
}
|
||||
});
|
||||
|
||||
test("static S3 Evidence resolves only configured secret-root file paths", async () => {
|
||||
test("static S3 Evidence resolves only ephemeral vault materializations", async () => {
|
||||
const f = await fixture(evidenceWorkspace(` type: s3
|
||||
uri: s3://clinical-evidence/published/
|
||||
endpoint_url: https://s3.example.test/
|
||||
@@ -361,7 +375,7 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
|
||||
retain_published_generations: 7
|
||||
`));
|
||||
const runner = runnerFor(f);
|
||||
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
const lease = await runner.acquireWorkspaceRuntime(f.revision.snapshotPath);
|
||||
try {
|
||||
const yaml = readFileSync(lease.path, "utf8");
|
||||
expect(parse(yaml).evidence.sources).toEqual([{
|
||||
@@ -370,9 +384,9 @@ test("static S3 Evidence resolves only configured secret-root file paths", async
|
||||
prefix: "published/",
|
||||
endpoint_url: "https://s3.example.test/",
|
||||
region: "eu-west-1",
|
||||
access_key_file: realpathSync(join(f.secretRoot, "evidence-access")),
|
||||
secret_key_file: realpathSync(join(f.secretRoot, "evidence-secret")),
|
||||
session_token_file: realpathSync(join(f.secretRoot, "evidence-token")),
|
||||
access_key_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
secret_key_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
session_token_file: expect.stringContaining("/workspace-secret-runtime/"),
|
||||
trusted_endpoint: true,
|
||||
allow_private_endpoint: true,
|
||||
allow_insecure_endpoint: false,
|
||||
|
||||
@@ -10,9 +10,9 @@ import {
|
||||
type SemanticRuntimeConfig,
|
||||
} from "../src/workspaces/runtime-renderer.js";
|
||||
import { supportsSessionRuntime } from "../src/workspaces/bindings.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV4 = parseWorkspaceYaml(`workspace:
|
||||
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -70,7 +70,14 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu
|
||||
},
|
||||
dwh: { type: "postgres_direct" },
|
||||
resources: {
|
||||
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
base_url: "http://qdrant:6333",
|
||||
collections: {
|
||||
reference: "psd-clinical-reference",
|
||||
memory: "psd-clinical-memory",
|
||||
},
|
||||
},
|
||||
embeddings: {
|
||||
provider: "ollama_internal", base_url: "http://embedding:11434",
|
||||
id: "ollama/qwen3-embedding:0.6b",
|
||||
@@ -133,7 +140,7 @@ function evidenceWorkspace(
|
||||
policy?: Record<string, unknown>,
|
||||
evidenceSchemaVersion?: number,
|
||||
) {
|
||||
return parseWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
|
||||
return parseRuntimeWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${
|
||||
evidenceSchemaVersion === undefined ? "" : `\n schema_version: ${evidenceSchemaVersion}`
|
||||
}\n source: ${JSON.stringify(source)}${
|
||||
policy === undefined ? "" : `\n policy: ${JSON.stringify(policy)}`
|
||||
|
||||
@@ -8,12 +8,12 @@ import {
|
||||
resolveRuntimeBindingsWithWorkspaceSecrets,
|
||||
} from "../src/workspaces/secret-requirements.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
function workspace(extra = "") {
|
||||
return parseWorkspaceYaml(`workspace:
|
||||
return parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
|
||||
@@ -23,14 +23,11 @@ test("strict workspace v4 rejects model-bearing v3 descriptors", () => {
|
||||
expect(() => parseWorkspaceYaml(legacy)).toThrow();
|
||||
});
|
||||
|
||||
test("v3 to v4 migration removes only model/vector policy and bumps the version", () => {
|
||||
test("v3 to v4 migration removes database/model policy and bumps the version", () => {
|
||||
const migrated = migrateWorkspaceV3Yaml(legacy);
|
||||
const workspace = parseWorkspaceYaml(migrated);
|
||||
|
||||
expect(workspace.workspace).toMatchObject({ schema_version: 4, id: "abc" });
|
||||
expect(migrated).not.toMatch(/semantic_index|llm_policy/);
|
||||
expect(workspace.dwh).toEqual({
|
||||
engine: "postgres", database: "warehouse", schema: "public",
|
||||
supported_transports: ["postgres_direct"],
|
||||
});
|
||||
expect(workspace.dwh).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -7,9 +7,9 @@ import { afterEach, expect, test } from "vitest";
|
||||
import {
|
||||
resolveBinding, resolveEvidenceBinding, resolveRuntimeBindings, supportsSessionRuntime,
|
||||
} from "../src/workspaces/bindings.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV4 = parseWorkspaceYaml(`workspace:
|
||||
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -63,7 +63,7 @@ test("requires workspace-v4 REST credentials unless the DWH diagnostic declares
|
||||
THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
|
||||
}, []).missing).toContain("THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE");
|
||||
|
||||
const noAuth = parseWorkspaceYaml(`workspace:
|
||||
const noAuth = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: No auth
|
||||
@@ -89,7 +89,7 @@ diagnostics:
|
||||
test("rejects unsupported transports and secret paths outside configured roots", () => {
|
||||
const outside = secretPath("outside-password");
|
||||
const allowed = secretPath("allowed-password");
|
||||
const directOnly = parseWorkspaceYaml(`workspace:
|
||||
const directOnly = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Direct only
|
||||
@@ -130,7 +130,7 @@ test("runtime bindings contain only DWH and Evidence roles", () => {
|
||||
});
|
||||
|
||||
function withEvidence(source: Record<string, unknown>) {
|
||||
return parseWorkspaceYaml(`workspace:
|
||||
return parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { expect, test } from "vitest";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import {
|
||||
CATALOG_PATH,
|
||||
assertCatalogMatchesDescriptor,
|
||||
parseWorkspaceCatalogYaml,
|
||||
} from "../src/workspaces/catalog.js";
|
||||
|
||||
const descriptor = parseWorkspaceYaml(`workspace:
|
||||
const descriptor = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd
|
||||
name: Policlinico San Donato
|
||||
|
||||
@@ -3,9 +3,9 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { buildInstallationContract, renderWorkspaceDocs } from "../src/workspaces/contracts.js";
|
||||
import { type CanonicalWorkspace, parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
import { type CanonicalWorkspace, parseRuntimeWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV4 = parseWorkspaceYaml(`workspace:
|
||||
const workspaceV4 = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -38,7 +38,7 @@ test.each([
|
||||
["rest_api", "BASE_URL", "HOST"],
|
||||
["ssh_tunnel", "SSH_PRIVATE_KEY_FILE", "BASE_URL"],
|
||||
] as const)("documents only the DWH fields for %s", (transport, included, excluded) => {
|
||||
const descriptor = parseWorkspaceYaml(renderWorkspaceWithoutEvidence()
|
||||
const descriptor = parseRuntimeWorkspaceYaml(renderWorkspaceWithoutEvidence()
|
||||
.replace("[postgres_direct]", `[${transport}]`));
|
||||
const variables = buildInstallationContract(descriptor).variables;
|
||||
expect(variables.find(({ suffix }) => suffix === included)?.transports).toEqual([transport]);
|
||||
@@ -87,7 +87,7 @@ test.each([
|
||||
],
|
||||
},
|
||||
])("generates source-specific $mode Evidence file bindings", ({ source, expected }) => {
|
||||
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
const contract = buildInstallationContract(descriptor);
|
||||
const evidence = contract.variables.filter((variable) => variable.role === "EVIDENCE");
|
||||
|
||||
@@ -102,7 +102,7 @@ test.each([
|
||||
{ type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" },
|
||||
{ type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" },
|
||||
])("omits Evidence installation variables for $type modes without file credentials", (source) => {
|
||||
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
const descriptor = parseRuntimeWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
|
||||
expect(buildInstallationContract(descriptor).variables.some((variable) => variable.role === "EVIDENCE"))
|
||||
.toBe(false);
|
||||
});
|
||||
@@ -151,7 +151,7 @@ const evidenceSources = [
|
||||
] as const;
|
||||
|
||||
test.each(evidenceSources)("renders deterministic public Evidence docs for $label", ({ source, variables }) => {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
|
||||
);
|
||||
const firstContract = buildInstallationContract(descriptor);
|
||||
@@ -178,7 +178,7 @@ test.each(evidenceSources)("renders deterministic public Evidence docs for $labe
|
||||
});
|
||||
|
||||
test("documents the S3 session token file as optional", () => {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:
|
||||
source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files }
|
||||
`,
|
||||
@@ -197,7 +197,7 @@ test("documents the S3 session token file as optional", () => {
|
||||
});
|
||||
|
||||
test("documents same-revision filesystem ownership without claiming P1 materialization", () => {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:\n source: { type: filesystem, uri: psd-clinical/evidence }\n`,
|
||||
);
|
||||
const docs = renderWorkspaceDocs(descriptor).markdown;
|
||||
@@ -236,7 +236,7 @@ test.each([
|
||||
const previous = process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE;
|
||||
process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = binding;
|
||||
try {
|
||||
const descriptor = parseWorkspaceYaml(
|
||||
const descriptor = parseRuntimeWorkspaceYaml(
|
||||
`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`,
|
||||
);
|
||||
const generated = JSON.stringify({
|
||||
|
||||
@@ -9,9 +9,9 @@ import {
|
||||
type DiagnosticAdapters,
|
||||
} from "../src/workspaces/diagnostics.js";
|
||||
import type { RuntimeBindings } from "../src/workspaces/runtime-renderer.js";
|
||||
import { parseWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
|
||||
import { parseRuntimeWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspace = parseWorkspaceYaml(`workspace:
|
||||
const workspace = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
@@ -81,9 +81,11 @@ test("diagnoses workspace-v4 DWH plus installation-derived Qdrant and Ollama", a
|
||||
role: "dwh", transport: "postgres_direct",
|
||||
resource: { database: "warehouse", schema: "datawarehouse" },
|
||||
}));
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({
|
||||
baseUrl: "http://qdrant:6333", collection: "psd-clinical",
|
||||
}));
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
|
||||
expect(vi.mocked(adapters.inspectQdrant).mock.calls.map(([request]) => request.collection)).toEqual([
|
||||
"psd-clinical-reference",
|
||||
"psd-clinical-memory",
|
||||
]);
|
||||
expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({
|
||||
baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b",
|
||||
}));
|
||||
@@ -107,14 +109,14 @@ test("can delegate the DWH probe to Database Management", async () => {
|
||||
}],
|
||||
});
|
||||
expect(adapters.probeConnector).not.toHaveBeenCalled();
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(1);
|
||||
expect(adapters.inspectQdrant).toHaveBeenCalledTimes(2);
|
||||
expect(adapters.probeEmbedding).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("reports incompatible internal Qdrant or Ollama metadata", async () => {
|
||||
const vector = await diagnose(successfulAdapters({
|
||||
inspectQdrant: vi.fn(async () => ({
|
||||
collection: "psd-clinical", dimensions: 768, distance: "cosine",
|
||||
inspectQdrant: vi.fn(async (request) => ({
|
||||
collection: request.collection, dimensions: 768, distance: "cosine",
|
||||
})),
|
||||
}))(workspace, bindings, { writeProbe: false });
|
||||
expect(vector.activatable).toBe(false);
|
||||
@@ -163,7 +165,7 @@ test("keeps workspace-v4 DWH SSH diagnostic-only and runtime-inactive", async ()
|
||||
});
|
||||
|
||||
test("uses the workspace-v4 declared DWH REST diagnostic and auth policy", async () => {
|
||||
const restWorkspace = parseWorkspaceYaml(`workspace:
|
||||
const restWorkspace = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: REST workspace
|
||||
@@ -428,7 +430,7 @@ test("uses a REST secret only as a header and redacts it from failed diagnostics
|
||||
const credentialFile = join(root, "api-key");
|
||||
const canary = "CANARY-REST-AUTH-SECRET";
|
||||
await writeFile(credentialFile, canary);
|
||||
const restDescriptor = parseWorkspaceYaml(`workspace:
|
||||
const restDescriptor = parseRuntimeWorkspaceYaml(`workspace:
|
||||
schema_version: 4
|
||||
id: psd-clinical
|
||||
name: REST auth
|
||||
@@ -548,6 +550,9 @@ test("returns observed normalized Qdrant distance for semantic mismatch classifi
|
||||
|
||||
test("classifies an observed non-cosine Qdrant distance as semantic incompatibility", async () => {
|
||||
const fetchMock = vi.fn()
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
|
||||
}), { status: 200 }))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
|
||||
}), { status: 200 }))
|
||||
|
||||
@@ -16,11 +16,6 @@ const validYaml = `workspace:
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports: [postgres_direct]
|
||||
`;
|
||||
|
||||
const runFile = promisify(execFile);
|
||||
|
||||
@@ -15,16 +15,6 @@ export const validYaml = `workspace:
|
||||
name: Policlinico San Donato
|
||||
description: Clinical data warehouse workspace
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
port: 5432
|
||||
timeout_ms: 5000
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
- rest_api
|
||||
- ssh_tunnel
|
||||
`;
|
||||
|
||||
test("rejects unknown keys and invalid immutable IDs", () => {
|
||||
@@ -41,21 +31,18 @@ test("rejects executable or otherwise custom YAML tags in canonical descriptors"
|
||||
))).toThrow(/tag|yaml/i);
|
||||
});
|
||||
|
||||
test("accepts optional connection ports and timeouts but rejects unsafe values", () => {
|
||||
expect(parseWorkspaceYaml(validYaml).dwh.port).toBe(5432);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 0")))
|
||||
.toThrow(/port/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 65536")))
|
||||
.toThrow(/port/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("timeout_ms: 5000", "timeout_ms: 0")))
|
||||
.toThrow(/timeout/i);
|
||||
test("rejects database configuration and diagnostics in authored workspace YAML", () => {
|
||||
expect(() => parseWorkspaceYaml(`${validYaml}dwh:\n engine: postgres\n database: d\n schema: s\n supported_transports: [postgres_direct]\n`))
|
||||
.toThrow(/must not contain database configuration/i);
|
||||
expect(() => parseWorkspaceYaml(`${validYaml}diagnostics:\n dwh_rest:\n method: GET\n path: \/health\n auth: none\n`))
|
||||
.toThrow(/must not contain database configuration/i);
|
||||
});
|
||||
|
||||
test("accepts a model-free schema v4 workspace", () => {
|
||||
test("accepts a database-free schema v4 workspace", () => {
|
||||
expect(parseWorkspaceYaml(validYaml)).toMatchObject({
|
||||
workspace: { schema_version: 4, id: "psd-clinical" },
|
||||
dwh: { database: "postgres", schema: "datawarehouse" },
|
||||
});
|
||||
expect(parseWorkspaceYaml(validYaml)).not.toHaveProperty("dwh");
|
||||
});
|
||||
|
||||
test("committed example descriptors parse as exact schema v4 workspaces", () => {
|
||||
@@ -137,7 +124,7 @@ llm_policy:
|
||||
- zai/glm-5.2
|
||||
`],
|
||||
])("rejects schema %s descriptors at parser and object-validator boundaries", (_version, yaml) => {
|
||||
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|4/i);
|
||||
expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|database configuration|4/i);
|
||||
expect(() => validateWorkspaceDescriptor(parse(yaml))).toThrow(/schema_version|invalid literal|4/i);
|
||||
});
|
||||
|
||||
@@ -159,7 +146,7 @@ test("constructs diagnostic URLs only when the resolved URL remains on the servi
|
||||
)).toThrow(/origin/i);
|
||||
});
|
||||
|
||||
test("rejects REST diagnostic declarations without their matching connector transport", () => {
|
||||
test("rejects REST diagnostic declarations in authored descriptors", () => {
|
||||
const diagnostics = `diagnostics:
|
||||
dwh_rest:
|
||||
method: POST
|
||||
@@ -171,7 +158,7 @@ test("rejects REST diagnostic declarations without their matching connector tran
|
||||
`;
|
||||
const declared = `${validYaml}${diagnostics}`;
|
||||
|
||||
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", ""))).toThrow(/dwh_rest/i);
|
||||
expect(() => parseWorkspaceYaml(declared)).toThrow(/database configuration/i);
|
||||
});
|
||||
|
||||
test("serializes canonical YAML that parses back to the same workspace", () => {
|
||||
|
||||
@@ -138,6 +138,30 @@ test("projects aggregate no-Evidence and completed-stage outcomes without rerunn
|
||||
expect(deps.persistJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("complete preprocessing preflights BM25 before continuing with Evidence", async () => {
|
||||
const deps = dependencies();
|
||||
deps.evidencePreflight.mockResolvedValue({
|
||||
ok: false as const,
|
||||
code: "semantic_index_incompatible" as const,
|
||||
});
|
||||
const state = job({ completedStages: ["catalog_snapshot", "schema_index"] });
|
||||
|
||||
const result = await continueEvidencePreprocessing(
|
||||
{ evidence: filesystemEvidence, job: state },
|
||||
deps,
|
||||
);
|
||||
|
||||
expect(deps.evidencePreflight).toHaveBeenCalledOnce();
|
||||
expect(deps.runStage).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
status: "failed",
|
||||
code: "semantic_index_incompatible",
|
||||
runId: "a".repeat(32),
|
||||
childRuns: {},
|
||||
completedStages: ["catalog_snapshot", "schema_index"],
|
||||
});
|
||||
});
|
||||
|
||||
test("preserves the narrow standalone projection for an already completed Evidence stage", async () => {
|
||||
const deps = dependencies();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user