feat: define internal semantic workspace schema
This commit is contained in:
@@ -102,7 +102,7 @@ export function resolveBinding(
|
||||
const supported = role === "DWH"
|
||||
? canonical.dwh.supported_transports
|
||||
: role === "VECTOR"
|
||||
? canonical.semantic_index.vector_store.supported_transports
|
||||
? (canonical.semantic_index.vector_store.supported_transports ?? [])
|
||||
: ["rest_api"] as const;
|
||||
const selectedValue = transportVariable ? env[transportVariable.name] : undefined;
|
||||
const selectedTransport = isTransport(selectedValue) ? selectedValue : supported[0];
|
||||
|
||||
@@ -131,7 +131,7 @@ export function buildInstallationContract(workspace: WorkspaceDescriptor): Insta
|
||||
...connectorVariables(
|
||||
namespace,
|
||||
"VECTOR",
|
||||
canonical.semantic_index.vector_store.supported_transports,
|
||||
canonical.semantic_index.vector_store.supported_transports ?? [],
|
||||
),
|
||||
...(canonical.semantic_index.vector_writer
|
||||
? [createVariable(namespace, "VECTOR_WRITER", "API_KEY_FILE")]
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { validateOperationalWorkspace, type WorkspaceV2, type WorkspaceV3 } from "./schema.js";
|
||||
|
||||
export function migrateWorkspaceV2ToV3(
|
||||
legacy: WorkspaceV2,
|
||||
collection: string,
|
||||
): WorkspaceV3 {
|
||||
return validateOperationalWorkspace({
|
||||
workspace: { ...legacy.workspace, schema_version: 3 },
|
||||
dwh: legacy.dwh,
|
||||
semantic_index: {
|
||||
vector_store: {
|
||||
engine: "qdrant",
|
||||
collection,
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
},
|
||||
embedding: {
|
||||
provider: "ollama_internal",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
},
|
||||
llm_policy: legacy.llm_policy,
|
||||
...(legacy.diagnostics?.dwh_rest
|
||||
? { diagnostics: { dwh_rest: legacy.diagnostics.dwh_rest } }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
@@ -99,8 +99,8 @@ export function renderRuntimeConfig(
|
||||
};
|
||||
const dwhIdentity = { database: canonical.dwh.database, schema: canonical.dwh.schema };
|
||||
const vectorIdentity = {
|
||||
database: canonical.semantic_index.vector_store.database,
|
||||
schema: canonical.semantic_index.vector_store.schema,
|
||||
database: canonical.semantic_index.vector_store.database ?? canonical.dwh.database,
|
||||
schema: canonical.semantic_index.vector_store.schema ?? canonical.dwh.schema,
|
||||
};
|
||||
const dwhDirect = bindings.dwh.transport === "postgres_direct";
|
||||
const vectorDirect = bindings.vector.transport === "pgvector_direct";
|
||||
|
||||
@@ -34,7 +34,7 @@ export interface CanonicalDiagnostics {
|
||||
embedding?: RestDiagnosticRequest & { response: { model: string; dimensions: string } };
|
||||
}
|
||||
|
||||
interface WorkspaceMetadata<Version extends 1 | 2> {
|
||||
interface WorkspaceMetadata<Version extends 1 | 2 | 3> {
|
||||
schema_version: Version;
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -72,23 +72,52 @@ interface SemanticIndex<TVectorStore extends VectorStore> {
|
||||
};
|
||||
}
|
||||
|
||||
interface WorkspaceBase<Version extends 1 | 2, TVectorStore extends VectorStore> {
|
||||
interface WorkspaceBase<Version extends 1 | 2 | 3, TVectorStore, TDiagnostics = CanonicalDiagnostics> {
|
||||
workspace: WorkspaceMetadata<Version>;
|
||||
dwh: WorkspaceDwh;
|
||||
semantic_index: SemanticIndex<TVectorStore>;
|
||||
semantic_index: {
|
||||
vector_store: TVectorStore;
|
||||
vector_writer?: Record<string, never>;
|
||||
embedding: Version extends 3 ? {
|
||||
provider: "ollama_internal";
|
||||
model: "qwen3-embedding:0.6b";
|
||||
dimensions: 1024;
|
||||
timeout_ms?: number;
|
||||
} : {
|
||||
provider: "ollama_compatible" | "openai_compatible";
|
||||
model: string;
|
||||
dimensions: number;
|
||||
timeout_ms?: number;
|
||||
};
|
||||
};
|
||||
llm_policy: {
|
||||
default?: `${string}/${string}`;
|
||||
allowed: `${string}/${string}`[];
|
||||
};
|
||||
diagnostics?: CanonicalDiagnostics;
|
||||
diagnostics?: TDiagnostics;
|
||||
}
|
||||
|
||||
export interface CanonicalWorkspace extends WorkspaceBase<2, VectorStore & { database: string; schema: string }> {}
|
||||
interface QdrantVectorStore {
|
||||
engine: "qdrant";
|
||||
collection: string;
|
||||
dimensions: 1024;
|
||||
distance: "cosine";
|
||||
database?: string;
|
||||
schema?: string;
|
||||
port?: number;
|
||||
timeout_ms?: number;
|
||||
supported_transports?: VectorTransport[];
|
||||
}
|
||||
|
||||
export interface WorkspaceV3 extends WorkspaceBase<3, QdrantVectorStore> {}
|
||||
export interface WorkspaceV2 extends WorkspaceBase<2, VectorStore & { database: string; schema: string }> {}
|
||||
|
||||
/** A readable, non-operational v1 descriptor. It must be explicitly migrated before use. */
|
||||
export interface LegacyWorkspace extends WorkspaceBase<1, VectorStore & { database?: string; schema?: string }> {}
|
||||
export interface WorkspaceV1 extends WorkspaceBase<1, VectorStore & { database?: string; schema?: string }> {}
|
||||
|
||||
export type WorkspaceDescriptor = CanonicalWorkspace | LegacyWorkspace;
|
||||
export type CanonicalWorkspace = WorkspaceV3;
|
||||
export type LegacyWorkspace = WorkspaceV1 | WorkspaceV2;
|
||||
export type WorkspaceDescriptor = WorkspaceV1 | WorkspaceV2 | WorkspaceV3;
|
||||
|
||||
const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, {
|
||||
message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$",
|
||||
@@ -159,6 +188,11 @@ const embeddingSchema = z.object({
|
||||
dimensions,
|
||||
timeout_ms: timeoutMs.optional(),
|
||||
}).strict();
|
||||
const internalEmbeddingSchema = z.object({
|
||||
provider: z.literal("ollama_internal"),
|
||||
model: z.literal("qwen3-embedding:0.6b"),
|
||||
dimensions: z.literal(1024),
|
||||
}).strict();
|
||||
const vectorStoreShape = {
|
||||
engine: z.literal("pgvector"),
|
||||
collection: identifier,
|
||||
@@ -178,6 +212,12 @@ const canonicalVectorStoreSchema = z.object({
|
||||
database: identifier,
|
||||
schema: identifier,
|
||||
}).strict();
|
||||
const qdrantVectorStoreSchema = z.object({
|
||||
engine: z.literal("qdrant"),
|
||||
collection: workspaceId,
|
||||
dimensions: z.literal(1024),
|
||||
distance: z.literal("cosine"),
|
||||
}).strict();
|
||||
const llmPolicySchema = z.object({
|
||||
default: modelReference.optional(),
|
||||
allowed: z.array(modelReference).min(1),
|
||||
@@ -191,11 +231,13 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert
|
||||
|
||||
function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
|
||||
unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
|
||||
unique(
|
||||
workspace.semantic_index.vector_store.supported_transports,
|
||||
context,
|
||||
["semantic_index", "vector_store", "supported_transports"],
|
||||
);
|
||||
if ("supported_transports" in workspace.semantic_index.vector_store) {
|
||||
unique(
|
||||
workspace.semantic_index.vector_store.supported_transports,
|
||||
context,
|
||||
["semantic_index", "vector_store", "supported_transports"],
|
||||
);
|
||||
}
|
||||
unique(workspace.llm_policy.allowed, context, ["llm_policy", "allowed"]);
|
||||
|
||||
if (workspace.semantic_index.vector_store.dimensions !== workspace.semantic_index.embedding.dimensions) {
|
||||
@@ -221,7 +263,10 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
|
||||
}
|
||||
if (
|
||||
workspace.diagnostics?.vector_rest
|
||||
&& !workspace.semantic_index.vector_store.supported_transports.includes("rest_api")
|
||||
&& (
|
||||
!("supported_transports" in workspace.semantic_index.vector_store)
|
||||
|| !workspace.semantic_index.vector_store.supported_transports.includes("rest_api")
|
||||
)
|
||||
) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
@@ -236,7 +281,7 @@ const workspaceShape = {
|
||||
llm_policy: llmPolicySchema,
|
||||
diagnostics: diagnosticsSchema,
|
||||
};
|
||||
const LegacyWorkspaceSchema = z.object({
|
||||
const WorkspaceV1Schema = z.object({
|
||||
...workspaceShape,
|
||||
workspace: z.object({
|
||||
schema_version: z.literal(1), id: workspaceId, name: z.string().trim().min(1),
|
||||
@@ -248,7 +293,7 @@ const LegacyWorkspaceSchema = z.object({
|
||||
embedding: embeddingSchema,
|
||||
}).strict(),
|
||||
}).strict().superRefine(workspaceInvariants);
|
||||
const CanonicalWorkspaceSchema = z.object({
|
||||
const WorkspaceV2Schema = z.object({
|
||||
...workspaceShape,
|
||||
workspace: z.object({
|
||||
schema_version: z.literal(2), id: workspaceId, name: z.string().trim().min(1),
|
||||
@@ -260,7 +305,22 @@ const CanonicalWorkspaceSchema = z.object({
|
||||
embedding: embeddingSchema,
|
||||
}).strict(),
|
||||
}).strict().superRefine(workspaceInvariants);
|
||||
const WorkspaceDescriptorSchema = z.union([CanonicalWorkspaceSchema, LegacyWorkspaceSchema]);
|
||||
const WorkspaceV3Schema = z.object({
|
||||
dwh: dwhSchema,
|
||||
llm_policy: llmPolicySchema,
|
||||
diagnostics: z.object({
|
||||
dwh_rest: dwhRestDiagnostic.optional(),
|
||||
}).strict().optional(),
|
||||
workspace: z.object({
|
||||
schema_version: z.literal(3), id: workspaceId, name: z.string().trim().min(1),
|
||||
description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]),
|
||||
}).strict(),
|
||||
semantic_index: z.object({
|
||||
vector_store: qdrantVectorStoreSchema,
|
||||
embedding: internalEmbeddingSchema,
|
||||
}).strict(),
|
||||
}).strict().superRefine(workspaceInvariants);
|
||||
const WorkspaceDescriptorSchema = z.union([WorkspaceV3Schema, WorkspaceV2Schema, WorkspaceV1Schema]);
|
||||
|
||||
export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
|
||||
const documents = parseAllDocuments(source, { uniqueKeys: true });
|
||||
@@ -278,14 +338,23 @@ export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescri
|
||||
}
|
||||
|
||||
export function isCanonicalWorkspace(workspace: WorkspaceDescriptor): workspace is CanonicalWorkspace {
|
||||
return workspace.workspace.schema_version === 2;
|
||||
return workspace.workspace.schema_version === 3;
|
||||
}
|
||||
|
||||
export function isOperationalWorkspace(workspace: WorkspaceDescriptor): workspace is WorkspaceV3 {
|
||||
return workspace.workspace.schema_version === 3;
|
||||
}
|
||||
|
||||
/** Rejects readable v1 descriptors at every operational boundary until a caller migrates them. */
|
||||
export function validateCanonicalWorkspace(workspace: unknown): CanonicalWorkspace {
|
||||
return validateOperationalWorkspace(workspace);
|
||||
}
|
||||
|
||||
/** Rejects readable v1/v2 descriptors at every operational boundary until a caller migrates them. */
|
||||
export function validateOperationalWorkspace(workspace: unknown): WorkspaceV3 {
|
||||
const descriptor = validateWorkspaceDescriptor(workspace);
|
||||
if (!isCanonicalWorkspace(descriptor)) {
|
||||
throw new Error("Workspace descriptor requires explicit migration to schema version 2");
|
||||
if (!isOperationalWorkspace(descriptor)) {
|
||||
throw new Error("Workspace descriptor requires explicit migration to schema version 3");
|
||||
}
|
||||
return descriptor;
|
||||
}
|
||||
@@ -295,19 +364,19 @@ export function validateCanonicalWorkspace(workspace: unknown): CanonicalWorkspa
|
||||
* transformer never derives it from DWH identity, even where both services share a database.
|
||||
*/
|
||||
export function migrateWorkspaceV1ToV2(
|
||||
workspace: LegacyWorkspace,
|
||||
workspace: WorkspaceV1,
|
||||
vectorIdentity: { database: string; schema: string },
|
||||
): CanonicalWorkspace {
|
||||
const legacy = LegacyWorkspaceSchema.parse(workspace) as LegacyWorkspace;
|
||||
): WorkspaceV2 {
|
||||
const legacy = WorkspaceV1Schema.parse(workspace) as WorkspaceV1;
|
||||
const identity = z.object({ database: identifier, schema: identifier }).strict().parse(vectorIdentity);
|
||||
return validateCanonicalWorkspace({
|
||||
return WorkspaceV2Schema.parse({
|
||||
...legacy,
|
||||
workspace: { ...legacy.workspace, schema_version: 2 },
|
||||
semantic_index: {
|
||||
...legacy.semantic_index,
|
||||
vector_store: { ...legacy.semantic_index.vector_store, ...identity },
|
||||
},
|
||||
});
|
||||
}) as WorkspaceV2;
|
||||
}
|
||||
|
||||
/** Builds a request URL only after rejecting values that can leave the declared service origin. */
|
||||
@@ -320,7 +389,7 @@ export function resolveDiagnosticUrl(baseUrl: string, path: string): URL {
|
||||
}
|
||||
|
||||
export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string {
|
||||
const canonical = validateCanonicalWorkspace(workspace);
|
||||
const canonical = validateOperationalWorkspace(workspace);
|
||||
return stringify(canonical, { lineWidth: 0, sortMapEntries: true });
|
||||
}
|
||||
|
||||
|
||||
@@ -15,3 +15,103 @@ export type WorkspaceErrorCode =
|
||||
| "workspace_stale" | "workspace_conflict" | "git_unavailable"
|
||||
| "git_auth_failed" | "git_non_fast_forward" | "git_push_rejected"
|
||||
| "connector_unavailable" | "semantic_index_incompatible";
|
||||
|
||||
export interface QdrantVectorStore {
|
||||
engine: "qdrant";
|
||||
collection: string;
|
||||
dimensions: 1024;
|
||||
distance: "cosine";
|
||||
database?: string;
|
||||
schema?: string;
|
||||
port?: number;
|
||||
timeout_ms?: number;
|
||||
supported_transports?: ("pgvector_direct" | "rest_api" | "ssh_tunnel")[];
|
||||
}
|
||||
|
||||
export interface InternalEmbedding {
|
||||
provider: "ollama_internal";
|
||||
model: "qwen3-embedding:0.6b";
|
||||
dimensions: 1024;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface WorkspaceV2 {
|
||||
workspace: {
|
||||
schema_version: 2;
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
language: "en" | "it";
|
||||
};
|
||||
dwh: {
|
||||
engine: "postgres";
|
||||
database: string;
|
||||
schema: string;
|
||||
port?: number;
|
||||
timeout_ms?: number;
|
||||
supported_transports: ("postgres_direct" | "rest_api" | "ssh_tunnel")[];
|
||||
};
|
||||
semantic_index: {
|
||||
vector_store: {
|
||||
engine: "pgvector";
|
||||
database: string;
|
||||
schema: string;
|
||||
collection: string;
|
||||
dimensions: number;
|
||||
distance: "cosine" | "l2" | "inner_product";
|
||||
port?: number;
|
||||
timeout_ms?: number;
|
||||
supported_transports: ("pgvector_direct" | "rest_api" | "ssh_tunnel")[];
|
||||
};
|
||||
vector_writer?: Record<string, never>;
|
||||
embedding: {
|
||||
provider: "ollama_compatible" | "openai_compatible";
|
||||
model: string;
|
||||
dimensions: number;
|
||||
timeout_ms?: number;
|
||||
};
|
||||
};
|
||||
llm_policy: {
|
||||
default?: `${string}/${string}`;
|
||||
allowed: `${string}/${string}`[];
|
||||
};
|
||||
diagnostics?: {
|
||||
dwh_rest?: {
|
||||
method: "GET" | "POST";
|
||||
path: string;
|
||||
auth: "none" | "bearer" | "x-api-key";
|
||||
response: { database: string; schema: string };
|
||||
};
|
||||
vector_rest?: {
|
||||
metadata: {
|
||||
method: "GET" | "POST";
|
||||
path: string;
|
||||
auth: "none" | "bearer" | "x-api-key";
|
||||
response: { collection: string; dimensions: string; distance: string };
|
||||
};
|
||||
reversible_probe?: {
|
||||
method: "POST";
|
||||
path: string;
|
||||
auth: "bearer" | "x-api-key";
|
||||
response: { operation: string };
|
||||
};
|
||||
};
|
||||
embedding?: {
|
||||
method: "GET" | "POST";
|
||||
path: string;
|
||||
auth: "none" | "bearer" | "x-api-key";
|
||||
response: { model: string; dimensions: string };
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export interface WorkspaceV3 {
|
||||
workspace: WorkspaceV2["workspace"] & { schema_version: 3 };
|
||||
dwh: WorkspaceV2["dwh"];
|
||||
semantic_index: {
|
||||
vector_store: QdrantVectorStore;
|
||||
embedding: InternalEmbedding;
|
||||
};
|
||||
llm_policy: WorkspaceV2["llm_policy"];
|
||||
diagnostics?: WorkspaceV2["diagnostics"];
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
migrateLegacyWorkspace,
|
||||
writeMigratedWorkspace,
|
||||
} from "../src/workspaces/migrate-legacy.js";
|
||||
import * as workspaceSchema from "../src/workspaces/schema.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const temporaryRoots: string[] = [];
|
||||
@@ -30,10 +31,15 @@ test("migrates the current local PSD descriptor without copying secret values",
|
||||
|
||||
test("keeps an incomplete legacy vector identity readable and explicitly migration-required", () => {
|
||||
const result = migrateLegacyWorkspace(readFixture("tht.example.yaml"), { id: "example" });
|
||||
const isOperationalWorkspace = (workspaceSchema as { isOperationalWorkspace?: unknown }).isOperationalWorkspace;
|
||||
|
||||
expect(result.state).toBe("migration_required");
|
||||
expect(result.workspace.workspace.schema_version).toBe(1);
|
||||
expect(parseWorkspaceYaml(result.source).workspace.schema_version).toBe(1);
|
||||
expect(isOperationalWorkspace).toBeTypeOf("function");
|
||||
expect((isOperationalWorkspace as (workspace: ReturnType<typeof parseWorkspaceYaml>) => boolean)(
|
||||
parseWorkspaceYaml(result.source),
|
||||
)).toBe(false);
|
||||
});
|
||||
|
||||
test("writes versioned repository artifacts atomically without replacing a prior migration", async () => {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { expect, test } from "vitest";
|
||||
import { migrateWorkspaceV2ToV3 } from "../src/workspaces/migrate-v2-qdrant.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV2Yaml = `workspace:
|
||||
schema_version: 2
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
description: Clinical data warehouse workspace
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
- rest_api
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
database: postgres
|
||||
schema: vectors
|
||||
collection: clinical_documents
|
||||
dimensions: 768
|
||||
distance: inner_product
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
- rest_api
|
||||
embedding:
|
||||
provider: openai_compatible
|
||||
model: text-embedding-3-large
|
||||
dimensions: 768
|
||||
llm_policy:
|
||||
default: zai/glm-5.2
|
||||
allowed:
|
||||
- zai/glm-5.2
|
||||
diagnostics:
|
||||
dwh_rest:
|
||||
method: POST
|
||||
path: /rpc/dwh
|
||||
auth: bearer
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /vector
|
||||
auth: bearer
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
embedding:
|
||||
method: GET
|
||||
path: /models
|
||||
auth: none
|
||||
response:
|
||||
model: model
|
||||
dimensions: dimensions
|
||||
`;
|
||||
|
||||
test("migrates a schema v2 workspace to the internal qdrant schema v3 shape", () => {
|
||||
const legacy = parseWorkspaceYaml(workspaceV2Yaml);
|
||||
|
||||
const migrated = migrateWorkspaceV2ToV3(legacy, "psd-clinical");
|
||||
|
||||
expect(migrated).toMatchObject({
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato" },
|
||||
dwh: legacy.dwh,
|
||||
semantic_index: {
|
||||
vector_store: {
|
||||
engine: "qdrant",
|
||||
collection: "psd-clinical",
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
},
|
||||
embedding: {
|
||||
provider: "ollama_internal",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
},
|
||||
llm_policy: legacy.llm_policy,
|
||||
diagnostics: {
|
||||
dwh_rest: legacy.diagnostics?.dwh_rest,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("drops vector and embedding diagnostics and transports during v2 to v3 migration", () => {
|
||||
const legacy = parseWorkspaceYaml(workspaceV2Yaml);
|
||||
|
||||
const migrated = migrateWorkspaceV2ToV3(legacy, "psd-clinical");
|
||||
|
||||
expect(migrated.diagnostics).toEqual({
|
||||
dwh_rest: legacy.diagnostics?.dwh_rest,
|
||||
});
|
||||
expect(migrated.semantic_index.vector_store).not.toHaveProperty("supported_transports");
|
||||
expect(migrated.semantic_index.embedding).not.toHaveProperty("timeout_ms");
|
||||
});
|
||||
@@ -1,9 +1,14 @@
|
||||
import { expect, test } from "vitest";
|
||||
import * as workspaceSchema from "../src/workspaces/schema.js";
|
||||
import { parseWorkspaceYaml, serializeWorkspaceYaml, validateCanonicalWorkspace } from "../src/workspaces/schema.js";
|
||||
import {
|
||||
parseWorkspaceYaml,
|
||||
serializeWorkspaceYaml,
|
||||
validateCanonicalWorkspace,
|
||||
type WorkspaceDescriptor,
|
||||
} from "../src/workspaces/schema.js";
|
||||
|
||||
export const validYaml = `workspace:
|
||||
schema_version: 2
|
||||
schema_version: 3
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
description: Clinical data warehouse workspace
|
||||
@@ -20,23 +25,14 @@ dwh:
|
||||
- ssh_tunnel
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
database: postgres
|
||||
schema: vectors
|
||||
collection: clinical_documents
|
||||
dimensions: 768
|
||||
engine: qdrant
|
||||
collection: psd-clinical
|
||||
dimensions: 1024
|
||||
distance: cosine
|
||||
port: 5432
|
||||
timeout_ms: 5000
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
- rest_api
|
||||
- ssh_tunnel
|
||||
embedding:
|
||||
provider: ollama_compatible
|
||||
model: nomic-embed-text-v2-moe
|
||||
dimensions: 768
|
||||
timeout_ms: 5000
|
||||
provider: ollama_internal
|
||||
model: qwen3-embedding:0.6b
|
||||
dimensions: 1024
|
||||
llm_policy:
|
||||
default: zai/glm-5.2
|
||||
allowed:
|
||||
@@ -45,7 +41,7 @@ llm_policy:
|
||||
`;
|
||||
|
||||
test("rejects a workspace whose embedding dimensions differ from its collection", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 768", "dimensions: 1536")))
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 1024", "dimensions: 1536")))
|
||||
.toThrow(/dimensions/i);
|
||||
});
|
||||
|
||||
@@ -76,113 +72,134 @@ test("accepts optional connection ports and timeouts but rejects unsafe values",
|
||||
.toThrow(/port/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("timeout_ms: 5000", "timeout_ms: 0")))
|
||||
.toThrow(/timeout/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 1024", "dimensions: 2048")))
|
||||
.toThrow(/1024|dimensions/i);
|
||||
});
|
||||
|
||||
test("requires explicit vector database and schema identities with strict diagnostic declarations", () => {
|
||||
const diagnosticWorkspace = validYaml.replace(
|
||||
" engine: pgvector\n database: postgres",
|
||||
" engine: pgvector\n database: vector_database",
|
||||
).replace(
|
||||
"llm_policy:\n",
|
||||
"diagnostics:\n"
|
||||
+ " dwh_rest:\n"
|
||||
+ " method: POST\n"
|
||||
+ " path: /rpc/ping\n"
|
||||
+ " auth: bearer\n"
|
||||
+ " response:\n"
|
||||
+ " database: database\n"
|
||||
+ " schema: schema\n"
|
||||
+ " vector_rest:\n"
|
||||
+ " metadata:\n"
|
||||
+ " method: GET\n"
|
||||
+ " path: /metadata\n"
|
||||
+ " auth: bearer\n"
|
||||
+ " response:\n"
|
||||
+ " collection: collection\n"
|
||||
+ " dimensions: dimensions\n"
|
||||
+ " distance: distance\n"
|
||||
+ " reversible_probe:\n"
|
||||
+ " method: POST\n"
|
||||
+ " path: /rpc/diagnostic_vector_probe\n"
|
||||
+ " auth: bearer\n"
|
||||
+ " response: { operation: operation }\n"
|
||||
+ " embedding:\n"
|
||||
+ " method: GET\n"
|
||||
+ " path: /models\n"
|
||||
+ " auth: none\n"
|
||||
+ " response:\n"
|
||||
+ " model: model\n"
|
||||
+ " dimensions: dimensions\n"
|
||||
+ "llm_policy:\n",
|
||||
);
|
||||
|
||||
expect(parseWorkspaceYaml(diagnosticWorkspace).semantic_index.vector_store).toMatchObject({
|
||||
database: "vector_database",
|
||||
schema: "vectors",
|
||||
test("accepts only the schema v3 internal qdrant semantic shape", () => {
|
||||
expect(parseWorkspaceYaml(validYaml)).toMatchObject({
|
||||
workspace: { schema_version: 3, id: "psd-clinical" },
|
||||
semantic_index: {
|
||||
vector_store: {
|
||||
engine: "qdrant",
|
||||
collection: "psd-clinical",
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
},
|
||||
embedding: {
|
||||
provider: "ollama_internal",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("database: vector_database", 'database: " "')))
|
||||
.toThrow(/database/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("schema: vectors", 'schema: " "')))
|
||||
.toThrow(/schema/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("method: POST", "method: PATCH")))
|
||||
.toThrow(/method/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace(" distance: distance", " distance: distance\n extra: ignored")))
|
||||
.toThrow(/unrecognized key/i);
|
||||
for (const unsafePath of [
|
||||
"//diagnostic.invalid/rpc", "'/\\\\diagnostic'", "'/rpc\\\\diagnostic'", "'/rpc/%5Cdiagnostic'", "'/rpc/\u0001'",
|
||||
]) {
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("path: /rpc/ping", `path: ${unsafePath}`)))
|
||||
.toThrow(/origin-relative|path/i);
|
||||
}
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("auth: bearer", "auth: basic")))
|
||||
.toThrow(/auth/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace(" schema: schema", " schema: schema\n status: status")))
|
||||
.toThrow(/unrecognized key/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace(" schema: schema", " schema: bad field")))
|
||||
.toThrow(/response field/i);
|
||||
});
|
||||
|
||||
test("requires a reversible writer probe to declare the response operation it verifies", () => {
|
||||
const writerProbe = validYaml.replace("llm_policy:\n", `diagnostics:
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: bearer
|
||||
response: { collection: collection, dimensions: dimensions, distance: distance }
|
||||
reversible_probe:
|
||||
method: POST
|
||||
path: /diagnostic-probe
|
||||
auth: bearer
|
||||
llm_policy:
|
||||
`);
|
||||
|
||||
expect(() => parseWorkspaceYaml(writerProbe)).toThrow(/response|operation/i);
|
||||
test("rejects pgvector semantic stores in schema v3", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("engine: qdrant", "engine: pgvector")))
|
||||
.toThrow(/qdrant|pgvector/i);
|
||||
});
|
||||
|
||||
test("keeps v1 descriptors readable but requires explicit migration before v2 operations", () => {
|
||||
const v1WithoutVectorIdentity = validYaml.replace("schema_version: 2", "schema_version: 1").replace(
|
||||
" database: postgres\n schema: vectors\n", "",
|
||||
test("rejects supported_transports inside schema v3 semantic identity", () => {
|
||||
const withTransport = validYaml.replace(
|
||||
" distance: cosine\n",
|
||||
" distance: cosine\n supported_transports:\n - rest_api\n",
|
||||
);
|
||||
expect(() => parseWorkspaceYaml(v1WithoutVectorIdentity)).not.toThrow();
|
||||
expect(() => parseWorkspaceYaml(validYaml)).not.toThrow();
|
||||
const v1 = parseWorkspaceYaml(v1WithoutVectorIdentity);
|
||||
const v2 = parseWorkspaceYaml(validYaml);
|
||||
|
||||
expect(() => parseWorkspaceYaml(withTransport)).toThrow(/unrecognized key|supported_transports/i);
|
||||
});
|
||||
|
||||
test("rejects external embedding providers in schema v3", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("provider: ollama_internal", "provider: openai_compatible")))
|
||||
.toThrow(/ollama_internal|provider/i);
|
||||
});
|
||||
|
||||
test("rejects non-cosine distance in schema v3", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("distance: cosine", "distance: l2")))
|
||||
.toThrow(/cosine|distance/i);
|
||||
});
|
||||
|
||||
test("rejects unknown fields in schema v3 semantic identity", () => {
|
||||
const withUnknownField = validYaml.replace(
|
||||
" collection: psd-clinical\n",
|
||||
" collection: psd-clinical\n namespace: psd\n",
|
||||
);
|
||||
|
||||
expect(() => parseWorkspaceYaml(withUnknownField)).toThrow(/unrecognized key/i);
|
||||
});
|
||||
|
||||
test("keeps v1 and v2 descriptors parseable but non-operational", () => {
|
||||
const v1Yaml = `workspace:
|
||||
schema_version: 1
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
collection: clinical_documents
|
||||
dimensions: 1024
|
||||
distance: cosine
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
embedding:
|
||||
provider: ollama_compatible
|
||||
model: nomic-embed-text-v2-moe
|
||||
dimensions: 1024
|
||||
llm_policy:
|
||||
allowed:
|
||||
- zai/glm-5.2
|
||||
`;
|
||||
const v2Yaml = `workspace:
|
||||
schema_version: 2
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
database: postgres
|
||||
schema: vectors
|
||||
collection: clinical_documents
|
||||
dimensions: 1024
|
||||
distance: cosine
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
embedding:
|
||||
provider: ollama_compatible
|
||||
model: nomic-embed-text-v2-moe
|
||||
dimensions: 1024
|
||||
llm_policy:
|
||||
allowed:
|
||||
- zai/glm-5.2
|
||||
`;
|
||||
|
||||
const v1 = parseWorkspaceYaml(v1Yaml);
|
||||
const v2 = parseWorkspaceYaml(v2Yaml);
|
||||
const isOperationalWorkspace = (workspaceSchema as { isOperationalWorkspace?: unknown }).isOperationalWorkspace;
|
||||
|
||||
expect(v1.workspace.schema_version).toBe(1);
|
||||
expect(() => validateCanonicalWorkspace(v1)).toThrow(/migrat/i);
|
||||
expect(v2.workspace.schema_version).toBe(2);
|
||||
|
||||
const migrate = (workspaceSchema as { migrateWorkspaceV1ToV2?: unknown }).migrateWorkspaceV1ToV2;
|
||||
expect(migrate).toBeTypeOf("function");
|
||||
const migrated = (migrate as (workspace: typeof v1, identity: { database: string; schema: string }) => unknown)(v1, {
|
||||
database: "vector_database",
|
||||
schema: "vectors",
|
||||
});
|
||||
expect(validateCanonicalWorkspace(migrated)).toMatchObject({
|
||||
workspace: { schema_version: 2 },
|
||||
semantic_index: { vector_store: { database: "vector_database", schema: "vectors" } },
|
||||
expect(validateCanonicalWorkspace(parseWorkspaceYaml(validYaml))).toMatchObject({
|
||||
workspace: { schema_version: 3 },
|
||||
});
|
||||
expect(() => validateCanonicalWorkspace(v1)).toThrow(/schema version 3|migration/i);
|
||||
expect(() => validateCanonicalWorkspace(v2)).toThrow(/schema version 3|migration/i);
|
||||
expect(isOperationalWorkspace).toBeTypeOf("function");
|
||||
expect((isOperationalWorkspace as (workspace: WorkspaceDescriptor) => boolean)(v1)).toBe(false);
|
||||
expect((isOperationalWorkspace as (workspace: WorkspaceDescriptor) => boolean)(v2)).toBe(false);
|
||||
});
|
||||
|
||||
test("constructs diagnostic URLs only when the resolved URL remains on the service origin", () => {
|
||||
@@ -205,23 +222,11 @@ test("rejects REST diagnostic declarations without their matching connector tran
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: bearer
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
llm_policy:
|
||||
`;
|
||||
const declared = validYaml.replace("llm_policy:\n", diagnostics);
|
||||
|
||||
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", ""))).toThrow(/dwh_rest/i);
|
||||
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", " - rest_api\n", 1).replace(
|
||||
" - rest_api\n", "",
|
||||
))).toThrow(/vector_rest/i);
|
||||
});
|
||||
|
||||
test("serializes canonical YAML that parses back to the same workspace", () => {
|
||||
|
||||
Reference in New Issue
Block a user