feat: define internal semantic workspace schema
This commit is contained in:
@@ -8,6 +8,7 @@ import {
|
||||
migrateLegacyWorkspace,
|
||||
writeMigratedWorkspace,
|
||||
} from "../src/workspaces/migrate-legacy.js";
|
||||
import * as workspaceSchema from "../src/workspaces/schema.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const temporaryRoots: string[] = [];
|
||||
@@ -30,10 +31,15 @@ test("migrates the current local PSD descriptor without copying secret values",
|
||||
|
||||
test("keeps an incomplete legacy vector identity readable and explicitly migration-required", () => {
|
||||
const result = migrateLegacyWorkspace(readFixture("tht.example.yaml"), { id: "example" });
|
||||
const isOperationalWorkspace = (workspaceSchema as { isOperationalWorkspace?: unknown }).isOperationalWorkspace;
|
||||
|
||||
expect(result.state).toBe("migration_required");
|
||||
expect(result.workspace.workspace.schema_version).toBe(1);
|
||||
expect(parseWorkspaceYaml(result.source).workspace.schema_version).toBe(1);
|
||||
expect(isOperationalWorkspace).toBeTypeOf("function");
|
||||
expect((isOperationalWorkspace as (workspace: ReturnType<typeof parseWorkspaceYaml>) => boolean)(
|
||||
parseWorkspaceYaml(result.source),
|
||||
)).toBe(false);
|
||||
});
|
||||
|
||||
test("writes versioned repository artifacts atomically without replacing a prior migration", async () => {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { expect, test } from "vitest";
|
||||
import { migrateWorkspaceV2ToV3 } from "../src/workspaces/migrate-v2-qdrant.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const workspaceV2Yaml = `workspace:
|
||||
schema_version: 2
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
description: Clinical data warehouse workspace
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
- rest_api
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
database: postgres
|
||||
schema: vectors
|
||||
collection: clinical_documents
|
||||
dimensions: 768
|
||||
distance: inner_product
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
- rest_api
|
||||
embedding:
|
||||
provider: openai_compatible
|
||||
model: text-embedding-3-large
|
||||
dimensions: 768
|
||||
llm_policy:
|
||||
default: zai/glm-5.2
|
||||
allowed:
|
||||
- zai/glm-5.2
|
||||
diagnostics:
|
||||
dwh_rest:
|
||||
method: POST
|
||||
path: /rpc/dwh
|
||||
auth: bearer
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /vector
|
||||
auth: bearer
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
embedding:
|
||||
method: GET
|
||||
path: /models
|
||||
auth: none
|
||||
response:
|
||||
model: model
|
||||
dimensions: dimensions
|
||||
`;
|
||||
|
||||
test("migrates a schema v2 workspace to the internal qdrant schema v3 shape", () => {
|
||||
const legacy = parseWorkspaceYaml(workspaceV2Yaml);
|
||||
|
||||
const migrated = migrateWorkspaceV2ToV3(legacy, "psd-clinical");
|
||||
|
||||
expect(migrated).toMatchObject({
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato" },
|
||||
dwh: legacy.dwh,
|
||||
semantic_index: {
|
||||
vector_store: {
|
||||
engine: "qdrant",
|
||||
collection: "psd-clinical",
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
},
|
||||
embedding: {
|
||||
provider: "ollama_internal",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
},
|
||||
llm_policy: legacy.llm_policy,
|
||||
diagnostics: {
|
||||
dwh_rest: legacy.diagnostics?.dwh_rest,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("drops vector and embedding diagnostics and transports during v2 to v3 migration", () => {
|
||||
const legacy = parseWorkspaceYaml(workspaceV2Yaml);
|
||||
|
||||
const migrated = migrateWorkspaceV2ToV3(legacy, "psd-clinical");
|
||||
|
||||
expect(migrated.diagnostics).toEqual({
|
||||
dwh_rest: legacy.diagnostics?.dwh_rest,
|
||||
});
|
||||
expect(migrated.semantic_index.vector_store).not.toHaveProperty("supported_transports");
|
||||
expect(migrated.semantic_index.embedding).not.toHaveProperty("timeout_ms");
|
||||
});
|
||||
@@ -1,9 +1,14 @@
|
||||
import { expect, test } from "vitest";
|
||||
import * as workspaceSchema from "../src/workspaces/schema.js";
|
||||
import { parseWorkspaceYaml, serializeWorkspaceYaml, validateCanonicalWorkspace } from "../src/workspaces/schema.js";
|
||||
import {
|
||||
parseWorkspaceYaml,
|
||||
serializeWorkspaceYaml,
|
||||
validateCanonicalWorkspace,
|
||||
type WorkspaceDescriptor,
|
||||
} from "../src/workspaces/schema.js";
|
||||
|
||||
export const validYaml = `workspace:
|
||||
schema_version: 2
|
||||
schema_version: 3
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
description: Clinical data warehouse workspace
|
||||
@@ -20,23 +25,14 @@ dwh:
|
||||
- ssh_tunnel
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
database: postgres
|
||||
schema: vectors
|
||||
collection: clinical_documents
|
||||
dimensions: 768
|
||||
engine: qdrant
|
||||
collection: psd-clinical
|
||||
dimensions: 1024
|
||||
distance: cosine
|
||||
port: 5432
|
||||
timeout_ms: 5000
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
- rest_api
|
||||
- ssh_tunnel
|
||||
embedding:
|
||||
provider: ollama_compatible
|
||||
model: nomic-embed-text-v2-moe
|
||||
dimensions: 768
|
||||
timeout_ms: 5000
|
||||
provider: ollama_internal
|
||||
model: qwen3-embedding:0.6b
|
||||
dimensions: 1024
|
||||
llm_policy:
|
||||
default: zai/glm-5.2
|
||||
allowed:
|
||||
@@ -45,7 +41,7 @@ llm_policy:
|
||||
`;
|
||||
|
||||
test("rejects a workspace whose embedding dimensions differ from its collection", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 768", "dimensions: 1536")))
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 1024", "dimensions: 1536")))
|
||||
.toThrow(/dimensions/i);
|
||||
});
|
||||
|
||||
@@ -76,113 +72,134 @@ test("accepts optional connection ports and timeouts but rejects unsafe values",
|
||||
.toThrow(/port/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("timeout_ms: 5000", "timeout_ms: 0")))
|
||||
.toThrow(/timeout/i);
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 1024", "dimensions: 2048")))
|
||||
.toThrow(/1024|dimensions/i);
|
||||
});
|
||||
|
||||
test("requires explicit vector database and schema identities with strict diagnostic declarations", () => {
|
||||
const diagnosticWorkspace = validYaml.replace(
|
||||
" engine: pgvector\n database: postgres",
|
||||
" engine: pgvector\n database: vector_database",
|
||||
).replace(
|
||||
"llm_policy:\n",
|
||||
"diagnostics:\n"
|
||||
+ " dwh_rest:\n"
|
||||
+ " method: POST\n"
|
||||
+ " path: /rpc/ping\n"
|
||||
+ " auth: bearer\n"
|
||||
+ " response:\n"
|
||||
+ " database: database\n"
|
||||
+ " schema: schema\n"
|
||||
+ " vector_rest:\n"
|
||||
+ " metadata:\n"
|
||||
+ " method: GET\n"
|
||||
+ " path: /metadata\n"
|
||||
+ " auth: bearer\n"
|
||||
+ " response:\n"
|
||||
+ " collection: collection\n"
|
||||
+ " dimensions: dimensions\n"
|
||||
+ " distance: distance\n"
|
||||
+ " reversible_probe:\n"
|
||||
+ " method: POST\n"
|
||||
+ " path: /rpc/diagnostic_vector_probe\n"
|
||||
+ " auth: bearer\n"
|
||||
+ " response: { operation: operation }\n"
|
||||
+ " embedding:\n"
|
||||
+ " method: GET\n"
|
||||
+ " path: /models\n"
|
||||
+ " auth: none\n"
|
||||
+ " response:\n"
|
||||
+ " model: model\n"
|
||||
+ " dimensions: dimensions\n"
|
||||
+ "llm_policy:\n",
|
||||
);
|
||||
|
||||
expect(parseWorkspaceYaml(diagnosticWorkspace).semantic_index.vector_store).toMatchObject({
|
||||
database: "vector_database",
|
||||
schema: "vectors",
|
||||
test("accepts only the schema v3 internal qdrant semantic shape", () => {
|
||||
expect(parseWorkspaceYaml(validYaml)).toMatchObject({
|
||||
workspace: { schema_version: 3, id: "psd-clinical" },
|
||||
semantic_index: {
|
||||
vector_store: {
|
||||
engine: "qdrant",
|
||||
collection: "psd-clinical",
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
},
|
||||
embedding: {
|
||||
provider: "ollama_internal",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("database: vector_database", 'database: " "')))
|
||||
.toThrow(/database/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("schema: vectors", 'schema: " "')))
|
||||
.toThrow(/schema/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("method: POST", "method: PATCH")))
|
||||
.toThrow(/method/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace(" distance: distance", " distance: distance\n extra: ignored")))
|
||||
.toThrow(/unrecognized key/i);
|
||||
for (const unsafePath of [
|
||||
"//diagnostic.invalid/rpc", "'/\\\\diagnostic'", "'/rpc\\\\diagnostic'", "'/rpc/%5Cdiagnostic'", "'/rpc/\u0001'",
|
||||
]) {
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("path: /rpc/ping", `path: ${unsafePath}`)))
|
||||
.toThrow(/origin-relative|path/i);
|
||||
}
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace("auth: bearer", "auth: basic")))
|
||||
.toThrow(/auth/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace(" schema: schema", " schema: schema\n status: status")))
|
||||
.toThrow(/unrecognized key/i);
|
||||
expect(() => parseWorkspaceYaml(diagnosticWorkspace.replace(" schema: schema", " schema: bad field")))
|
||||
.toThrow(/response field/i);
|
||||
});
|
||||
|
||||
test("requires a reversible writer probe to declare the response operation it verifies", () => {
|
||||
const writerProbe = validYaml.replace("llm_policy:\n", `diagnostics:
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: bearer
|
||||
response: { collection: collection, dimensions: dimensions, distance: distance }
|
||||
reversible_probe:
|
||||
method: POST
|
||||
path: /diagnostic-probe
|
||||
auth: bearer
|
||||
llm_policy:
|
||||
`);
|
||||
|
||||
expect(() => parseWorkspaceYaml(writerProbe)).toThrow(/response|operation/i);
|
||||
test("rejects pgvector semantic stores in schema v3", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("engine: qdrant", "engine: pgvector")))
|
||||
.toThrow(/qdrant|pgvector/i);
|
||||
});
|
||||
|
||||
test("keeps v1 descriptors readable but requires explicit migration before v2 operations", () => {
|
||||
const v1WithoutVectorIdentity = validYaml.replace("schema_version: 2", "schema_version: 1").replace(
|
||||
" database: postgres\n schema: vectors\n", "",
|
||||
test("rejects supported_transports inside schema v3 semantic identity", () => {
|
||||
const withTransport = validYaml.replace(
|
||||
" distance: cosine\n",
|
||||
" distance: cosine\n supported_transports:\n - rest_api\n",
|
||||
);
|
||||
expect(() => parseWorkspaceYaml(v1WithoutVectorIdentity)).not.toThrow();
|
||||
expect(() => parseWorkspaceYaml(validYaml)).not.toThrow();
|
||||
const v1 = parseWorkspaceYaml(v1WithoutVectorIdentity);
|
||||
const v2 = parseWorkspaceYaml(validYaml);
|
||||
|
||||
expect(() => parseWorkspaceYaml(withTransport)).toThrow(/unrecognized key|supported_transports/i);
|
||||
});
|
||||
|
||||
test("rejects external embedding providers in schema v3", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("provider: ollama_internal", "provider: openai_compatible")))
|
||||
.toThrow(/ollama_internal|provider/i);
|
||||
});
|
||||
|
||||
test("rejects non-cosine distance in schema v3", () => {
|
||||
expect(() => parseWorkspaceYaml(validYaml.replace("distance: cosine", "distance: l2")))
|
||||
.toThrow(/cosine|distance/i);
|
||||
});
|
||||
|
||||
test("rejects unknown fields in schema v3 semantic identity", () => {
|
||||
const withUnknownField = validYaml.replace(
|
||||
" collection: psd-clinical\n",
|
||||
" collection: psd-clinical\n namespace: psd\n",
|
||||
);
|
||||
|
||||
expect(() => parseWorkspaceYaml(withUnknownField)).toThrow(/unrecognized key/i);
|
||||
});
|
||||
|
||||
test("keeps v1 and v2 descriptors parseable but non-operational", () => {
|
||||
const v1Yaml = `workspace:
|
||||
schema_version: 1
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
collection: clinical_documents
|
||||
dimensions: 1024
|
||||
distance: cosine
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
embedding:
|
||||
provider: ollama_compatible
|
||||
model: nomic-embed-text-v2-moe
|
||||
dimensions: 1024
|
||||
llm_policy:
|
||||
allowed:
|
||||
- zai/glm-5.2
|
||||
`;
|
||||
const v2Yaml = `workspace:
|
||||
schema_version: 2
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: it
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports:
|
||||
- postgres_direct
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: pgvector
|
||||
database: postgres
|
||||
schema: vectors
|
||||
collection: clinical_documents
|
||||
dimensions: 1024
|
||||
distance: cosine
|
||||
supported_transports:
|
||||
- pgvector_direct
|
||||
embedding:
|
||||
provider: ollama_compatible
|
||||
model: nomic-embed-text-v2-moe
|
||||
dimensions: 1024
|
||||
llm_policy:
|
||||
allowed:
|
||||
- zai/glm-5.2
|
||||
`;
|
||||
|
||||
const v1 = parseWorkspaceYaml(v1Yaml);
|
||||
const v2 = parseWorkspaceYaml(v2Yaml);
|
||||
const isOperationalWorkspace = (workspaceSchema as { isOperationalWorkspace?: unknown }).isOperationalWorkspace;
|
||||
|
||||
expect(v1.workspace.schema_version).toBe(1);
|
||||
expect(() => validateCanonicalWorkspace(v1)).toThrow(/migrat/i);
|
||||
expect(v2.workspace.schema_version).toBe(2);
|
||||
|
||||
const migrate = (workspaceSchema as { migrateWorkspaceV1ToV2?: unknown }).migrateWorkspaceV1ToV2;
|
||||
expect(migrate).toBeTypeOf("function");
|
||||
const migrated = (migrate as (workspace: typeof v1, identity: { database: string; schema: string }) => unknown)(v1, {
|
||||
database: "vector_database",
|
||||
schema: "vectors",
|
||||
});
|
||||
expect(validateCanonicalWorkspace(migrated)).toMatchObject({
|
||||
workspace: { schema_version: 2 },
|
||||
semantic_index: { vector_store: { database: "vector_database", schema: "vectors" } },
|
||||
expect(validateCanonicalWorkspace(parseWorkspaceYaml(validYaml))).toMatchObject({
|
||||
workspace: { schema_version: 3 },
|
||||
});
|
||||
expect(() => validateCanonicalWorkspace(v1)).toThrow(/schema version 3|migration/i);
|
||||
expect(() => validateCanonicalWorkspace(v2)).toThrow(/schema version 3|migration/i);
|
||||
expect(isOperationalWorkspace).toBeTypeOf("function");
|
||||
expect((isOperationalWorkspace as (workspace: WorkspaceDescriptor) => boolean)(v1)).toBe(false);
|
||||
expect((isOperationalWorkspace as (workspace: WorkspaceDescriptor) => boolean)(v2)).toBe(false);
|
||||
});
|
||||
|
||||
test("constructs diagnostic URLs only when the resolved URL remains on the service origin", () => {
|
||||
@@ -205,23 +222,11 @@ test("rejects REST diagnostic declarations without their matching connector tran
|
||||
response:
|
||||
database: database
|
||||
schema: schema
|
||||
vector_rest:
|
||||
metadata:
|
||||
method: GET
|
||||
path: /metadata
|
||||
auth: bearer
|
||||
response:
|
||||
collection: collection
|
||||
dimensions: dimensions
|
||||
distance: distance
|
||||
llm_policy:
|
||||
`;
|
||||
const declared = validYaml.replace("llm_policy:\n", diagnostics);
|
||||
|
||||
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", ""))).toThrow(/dwh_rest/i);
|
||||
expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", " - rest_api\n", 1).replace(
|
||||
" - rest_api\n", "",
|
||||
))).toThrow(/vector_rest/i);
|
||||
});
|
||||
|
||||
test("serializes canonical YAML that parses back to the same workspace", () => {
|
||||
|
||||
Reference in New Issue
Block a user