diff --git a/backend/src/workspaces/bindings.ts b/backend/src/workspaces/bindings.ts index 042f3cf1..a3c26157 100644 --- a/backend/src/workspaces/bindings.ts +++ b/backend/src/workspaces/bindings.ts @@ -63,7 +63,7 @@ function isSafeSecretFile(path: string, secretRoots: readonly string[]): boolean } function requiredSuffixes( - role: InstallationRole, + role: Exclude, transport: DwhTransport | VectorTransport, ): readonly InstallationSuffix[] { if (role === "EMBEDDING") return EMBEDDING_REQUIRED_SUFFIXES; @@ -76,7 +76,7 @@ function requiredSuffixes( */ export function resolveBinding( workspace: CanonicalWorkspace, - role: InstallationRole, + role: Exclude, env: NodeJS.ProcessEnv, secretRoots: readonly string[], ): ResolvedBinding { diff --git a/backend/src/workspaces/contracts.ts b/backend/src/workspaces/contracts.ts index 0d97d23a..f2252ecb 100644 --- a/backend/src/workspaces/contracts.ts +++ b/backend/src/workspaces/contracts.ts @@ -1,7 +1,7 @@ import { validateCanonicalWorkspace } from "./schema.js"; import type { CanonicalWorkspace, DwhTransport, VectorTransport } from "./schema.js"; -export type InstallationRole = "DWH" | "VECTOR" | "EMBEDDING"; +export type InstallationRole = "DWH" | "VECTOR" | "VECTOR_WRITER" | "EMBEDDING"; export type InstallationSuffix = | "TRANSPORT" | "HOST" @@ -133,6 +133,9 @@ export function buildInstallationContract(workspace: CanonicalWorkspace): Instal "VECTOR", canonical.semantic_index.vector_store.supported_transports, ), + ...(canonical.semantic_index.vector_writer + ? [createVariable(namespace, "VECTOR_WRITER", "API_KEY_FILE")] + : []), ...EMBEDDING_SUFFIXES.map((suffix) => createVariable(namespace, "EMBEDDING", suffix)), ], }; @@ -170,8 +173,10 @@ export function renderWorkspaceDocs(workspace: CanonicalWorkspace): { envExample "", "Use the following UI fields as installation bindings. Secret fields always contain file paths, never secret values.", "", - ...(["DWH", "VECTOR", "EMBEDDING"] as const).flatMap((role) => [ - `## ${role === "DWH" ? "Data warehouse" : role === "VECTOR" ? "Vector store" : "Embedding service"}`, + ...(["DWH", "VECTOR", "VECTOR_WRITER", "EMBEDDING"] as const) + .filter((role) => variablesByRole.has(role)) + .flatMap((role) => [ + `## ${role === "DWH" ? "Data warehouse" : role === "VECTOR" ? "Vector store" : role === "VECTOR_WRITER" ? "Vector writer" : "Embedding service"}`, "", ...(variablesByRole.get(role) ?? []).map((variable) => ( `- \`${variable.name}\`${variable.transports ? ` (for: ${variable.transports.join(", ")})` : ""}` diff --git a/backend/src/workspaces/runtime-renderer.ts b/backend/src/workspaces/runtime-renderer.ts index e6db2960..58bb2311 100644 --- a/backend/src/workspaces/runtime-renderer.ts +++ b/backend/src/workspaces/runtime-renderer.ts @@ -90,7 +90,10 @@ export function renderRuntimeConfig( return variable.name; }; const dwhIdentity = { database: canonical.dwh.database, schema: canonical.dwh.schema }; - const vectorIdentity = dwhIdentity; + const vectorIdentity = { + database: canonical.semantic_index.vector_store.database, + schema: canonical.semantic_index.vector_store.schema, + }; const dwhDirect = bindings.dwh.transport === "postgres_direct"; const vectorDirect = bindings.vector.transport === "pgvector_direct"; const database = dwhDirect diff --git a/backend/src/workspaces/schema.ts b/backend/src/workspaces/schema.ts index 619ca55c..5cab72cc 100644 --- a/backend/src/workspaces/schema.ts +++ b/backend/src/workspaces/schema.ts @@ -7,6 +7,33 @@ export type DwhTransport = (typeof DWH_TRANSPORTS)[number]; export const VECTOR_TRANSPORTS = ["pgvector_direct", "rest_api", "ssh_tunnel"] as const; export type VectorTransport = (typeof VECTOR_TRANSPORTS)[number]; +export const REST_DIAGNOSTIC_METHODS = ["GET", "POST"] as const; +export type RestDiagnosticMethod = (typeof REST_DIAGNOSTIC_METHODS)[number]; + +export const DIAGNOSTIC_AUTH_MODES = ["none", "bearer"] as const; +export type DiagnosticAuthMode = (typeof DIAGNOSTIC_AUTH_MODES)[number]; + +export interface RestDiagnosticRequest { + method: RestDiagnosticMethod; + path: string; + auth: DiagnosticAuthMode; +} + +export interface CanonicalDiagnostics { + dwh_rest?: RestDiagnosticRequest & { + response: { database: string; schema: string }; + }; + vector_rest?: { + metadata: RestDiagnosticRequest & { + response: { collection: string; dimensions: string; distance: string }; + }; + reversible_probe?: RestDiagnosticRequest & { method: "POST" }; + }; + embedding?: RestDiagnosticRequest & { + response: { model: string; dimensions: string }; + }; +} + export interface CanonicalWorkspace { workspace: { schema_version: 1; @@ -26,6 +53,8 @@ export interface CanonicalWorkspace { semantic_index: { vector_store: { engine: "pgvector"; + database: string; + schema: string; collection: string; dimensions: number; distance: "cosine" | "l2" | "inner_product"; @@ -33,6 +62,7 @@ export interface CanonicalWorkspace { timeout_ms?: number; supported_transports: VectorTransport[]; }; + vector_writer?: Record; embedding: { provider: "ollama_compatible" | "openai_compatible"; model: string; @@ -44,6 +74,7 @@ export interface CanonicalWorkspace { default?: `${string}/${string}`; allowed: `${string}/${string}`[]; }; + diagnostics?: CanonicalDiagnostics; } const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, { @@ -58,6 +89,31 @@ const timeoutMs = z.number().int().positive(); const modelReference = z.string().regex(/^[^/\s]+\/[^/\s]+$/, { message: "model must use provider/model syntax", }); +const diagnosticPath = z.string().regex(/^\/[^\s?#]*$/, { + message: "diagnostic paths must be absolute and cannot include whitespace, queries, or fragments", +}); +const responseField = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, { + message: "diagnostic response fields must be identifiers", +}); +const restDiagnosticRequest = z.object({ + method: z.enum(REST_DIAGNOSTIC_METHODS), + path: diagnosticPath, + auth: z.enum(DIAGNOSTIC_AUTH_MODES), +}).strict(); +const dwhRestDiagnostic = restDiagnosticRequest.extend({ + response: z.object({ database: responseField, schema: responseField }).strict(), +}).strict(); +const vectorMetadataDiagnostic = restDiagnosticRequest.extend({ + response: z.object({ + collection: responseField, + dimensions: responseField, + distance: responseField, + }).strict(), +}).strict(); +const reversibleVectorProbe = restDiagnosticRequest.extend({ method: z.literal("POST") }).strict(); +const embeddingDiagnostic = restDiagnosticRequest.extend({ + response: z.object({ model: responseField, dimensions: responseField }).strict(), +}).strict(); function unique(values: readonly T[], context: z.RefinementCtx, path: PropertyKey[]) { if (new Set(values).size !== values.length) { @@ -84,6 +140,8 @@ const WorkspaceSchema = z.object({ semantic_index: z.object({ vector_store: z.object({ engine: z.literal("pgvector"), + database: identifier, + schema: identifier, collection: identifier, dimensions, distance: z.enum(["cosine", "l2", "inner_product"]), @@ -91,6 +149,7 @@ const WorkspaceSchema = z.object({ timeout_ms: timeoutMs.optional(), supported_transports: z.array(z.enum(VECTOR_TRANSPORTS)).min(1), }).strict(), + vector_writer: z.object({}).strict().optional(), embedding: z.object({ provider: z.enum(["ollama_compatible", "openai_compatible"]), model: z.string().trim().min(1), @@ -102,6 +161,14 @@ const WorkspaceSchema = z.object({ default: modelReference.optional(), allowed: z.array(modelReference).min(1), }).strict(), + diagnostics: z.object({ + dwh_rest: dwhRestDiagnostic.optional(), + vector_rest: z.object({ + metadata: vectorMetadataDiagnostic, + reversible_probe: reversibleVectorProbe.optional(), + }).strict().optional(), + embedding: embeddingDiagnostic.optional(), + }).strict().optional(), }).strict().superRefine((workspace, context) => { unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]); unique( diff --git a/backend/test/workspace-registry.test.ts b/backend/test/workspace-registry.test.ts index 18a42428..4ba84cf4 100644 --- a/backend/test/workspace-registry.test.ts +++ b/backend/test/workspace-registry.test.ts @@ -23,6 +23,8 @@ dwh: semantic_index: vector_store: engine: pgvector + database: postgres + schema: vectors collection: clinical_documents dimensions: 768 distance: cosine diff --git a/backend/test/workspace-runtime-renderer.test.ts b/backend/test/workspace-runtime-renderer.test.ts index e0a047fd..7315f629 100644 --- a/backend/test/workspace-runtime-renderer.test.ts +++ b/backend/test/workspace-runtime-renderer.test.ts @@ -16,6 +16,8 @@ dwh: semantic_index: vector_store: engine: pgvector + database: postgres + schema: vectors collection: clinical_documents dimensions: 768 distance: cosine @@ -81,7 +83,8 @@ test("renders a direct PostgreSQL binding to the legacy harness shape", () => { }, vector_db: { host: "vector.internal", - schema: "datawarehouse", + database: "postgres", + schema: "vectors", password_file: "/run/secrets/vector-password", ssl_ca_file: "/run/secrets/vector-ca.pem", }, diff --git a/backend/test/workspaces-bindings.test.ts b/backend/test/workspaces-bindings.test.ts index fbfee982..31d8786e 100644 --- a/backend/test/workspaces-bindings.test.ts +++ b/backend/test/workspaces-bindings.test.ts @@ -18,6 +18,8 @@ dwh: semantic_index: vector_store: engine: pgvector + database: postgres + schema: vectors collection: clinical_documents dimensions: 768 distance: cosine diff --git a/backend/test/workspaces-contracts.test.ts b/backend/test/workspaces-contracts.test.ts index ac7acb81..00b2a927 100644 --- a/backend/test/workspaces-contracts.test.ts +++ b/backend/test/workspaces-contracts.test.ts @@ -1,6 +1,8 @@ import { expect, test } from "vitest"; +import { parse } from "yaml"; import { buildInstallationContract, renderWorkspaceDocs } from "../src/workspaces/contracts.js"; import { type CanonicalWorkspace, parseWorkspaceYaml } from "../src/workspaces/schema.js"; +import { renderRuntimeConfig, type RuntimeBindings } from "../src/workspaces/runtime-renderer.js"; const validWorkspace = parseWorkspaceYaml(`workspace: schema_version: 1 @@ -17,6 +19,8 @@ dwh: semantic_index: vector_store: engine: pgvector + database: postgres + schema: vectors collection: clinical_documents dimensions: 768 distance: cosine @@ -64,9 +68,108 @@ test("renders English UI headings and workspace-language Italian prose", () => { expect(docs.markdown).toContain("# Installation requirements"); expect(docs.markdown).toContain("Configurazione dell'installazione"); + expect(docs.markdown).not.toContain("## Vector writer"); expect(docs.envExample).toContain("THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT="); }); +test("renders the vector store identity and creates writer credentials only when declared", () => { + const writerWorkspace = parseWorkspaceYaml(`workspace: + schema_version: 1 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: warehouse + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: pgvector + database: vector_database + schema: vectors + collection: clinical_documents + dimensions: 768 + distance: cosine + supported_transports: [pgvector_direct] + vector_writer: {} + embedding: + provider: ollama_compatible + model: nomic-embed-text-v2-moe + dimensions: 768 +diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: bearer + response: + database: database + schema: schema + vector_rest: + metadata: + method: GET + path: /metadata + auth: bearer + response: + collection: collection + dimensions: dimensions + distance: distance + embedding: + method: GET + path: /models + auth: none + response: + model: model + dimensions: dimensions +llm_policy: + allowed: [zai/glm-5.2] +`); + const bindings: RuntimeBindings = { + dwh: { + transport: "postgres_direct", + missing: [], + values: { + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: "/run/secrets/dwh", + }, + }, + vector: { + transport: "pgvector_direct", + missing: [], + values: { + THT_WS_PSD_CLINICAL_VECTOR_HOST: "vector.internal", + THT_WS_PSD_CLINICAL_VECTOR_PORT: "5432", + THT_WS_PSD_CLINICAL_VECTOR_USER: "vector-reader", + THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE: "/run/secrets/vector-reader", + }, + }, + embedding: { + transport: "rest_api", + missing: [], + values: { THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL: "https://embedding.internal" }, + }, + }; + + const writerVariables = buildInstallationContract(writerWorkspace).variables + .filter((variable) => variable.role === "VECTOR_WRITER"); + + expect(writerVariables).toEqual([expect.objectContaining({ + name: "THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE", + role: "VECTOR_WRITER", + secret: true, + })]); + expect(buildInstallationContract(validWorkspace).variables.some((variable) => ( + variable.role === "VECTOR_WRITER" + ))).toBe(false); + expect(parse(renderRuntimeConfig(writerWorkspace, bindings, { + sessions: "/data/sessions", + artifacts: "/data/artifacts", + indexes: "/data/indexes", + })).vector_db).toMatchObject({ database: "vector_database", schema: "vectors" }); +}); + function withTransports( dwhTransport: CanonicalWorkspace["dwh"]["supported_transports"][number], vectorTransport: CanonicalWorkspace["semantic_index"]["vector_store"]["supported_transports"][number], diff --git a/backend/test/workspaces-diagnostics.test.ts b/backend/test/workspaces-diagnostics.test.ts index 0ea7dd1b..092ceb03 100644 --- a/backend/test/workspaces-diagnostics.test.ts +++ b/backend/test/workspaces-diagnostics.test.ts @@ -21,6 +21,8 @@ dwh: semantic_index: vector_store: engine: pgvector + database: postgres + schema: vectors collection: clinical_documents dimensions: 768 distance: cosine diff --git a/backend/test/workspaces-git-repository.test.ts b/backend/test/workspaces-git-repository.test.ts index c656fc99..2fc79c7f 100644 --- a/backend/test/workspaces-git-repository.test.ts +++ b/backend/test/workspaces-git-repository.test.ts @@ -20,6 +20,8 @@ dwh: semantic_index: vector_store: engine: pgvector + database: postgres + schema: vectors collection: clinical_documents dimensions: 768 distance: cosine diff --git a/backend/test/workspaces-schema.test.ts b/backend/test/workspaces-schema.test.ts index 8f48d79e..3f79cdc3 100644 --- a/backend/test/workspaces-schema.test.ts +++ b/backend/test/workspaces-schema.test.ts @@ -20,6 +20,8 @@ dwh: semantic_index: vector_store: engine: pgvector + database: postgres + schema: vectors collection: clinical_documents dimensions: 768 distance: cosine @@ -68,6 +70,57 @@ test("accepts optional connection ports and timeouts but rejects unsafe values", .toThrow(/timeout/i); }); +test("requires explicit vector database and schema identities with strict diagnostic declarations", () => { + const diagnosticWorkspace = validYaml.replace( + " engine: pgvector\n database: postgres", + " engine: pgvector\n database: vector_database", + ).replace( + "llm_policy:\n", + "diagnostics:\n" + + " dwh_rest:\n" + + " method: POST\n" + + " path: /rpc/ping\n" + + " auth: bearer\n" + + " response:\n" + + " database: database\n" + + " schema: schema\n" + + " vector_rest:\n" + + " metadata:\n" + + " method: GET\n" + + " path: /metadata\n" + + " auth: bearer\n" + + " response:\n" + + " collection: collection\n" + + " dimensions: dimensions\n" + + " distance: distance\n" + + " reversible_probe:\n" + + " method: POST\n" + + " path: /rpc/diagnostic_vector_probe\n" + + " auth: bearer\n" + + " embedding:\n" + + " method: GET\n" + + " path: /models\n" + + " auth: none\n" + + " response:\n" + + " model: model\n" + + " dimensions: dimensions\n" + + "llm_policy:\n", + ); + + expect(parseWorkspaceYaml(diagnosticWorkspace).semantic_index.vector_store).toMatchObject({ + database: "vector_database", + schema: "vectors", + }); + expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("database: vector_database", 'database: " "'))) + .toThrow(/database/i); + expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("schema: vectors", 'schema: " "'))) + .toThrow(/schema/i); + expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("method: POST", "method: PATCH"))) + .toThrow(/method/i); + expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace(" distance: distance", " distance: distance\n extra: ignored"))) + .toThrow(/unrecognized key/i); +}); + test("serializes canonical YAML that parses back to the same workspace", () => { const workspace = parseWorkspaceYaml(validYaml); const serialized = serializeWorkspaceYaml(workspace);