mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: define internal semantic workspace schema

Browse Source
This commit is contained in:
marcopan
2026-08-08 16:43:54 +02:00
parent 4fe4049a24
commit 2f7f923c4d
9 changed files with 463 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/workspaces/bindings.ts
+1 -1
View File
@@ -102,7 +102,7 @@ export function resolveBinding(
const supported = role === "DWH"
? canonical.dwh.supported_transports
: role === "VECTOR"
? canonical.semantic_index.vector_store.supported_transports
? (canonical.semantic_index.vector_store.supported_transports ?? [])
: ["rest_api"] as const;
const selectedValue = transportVariable ? env[transportVariable.name] : undefined;
const selectedTransport = isTransport(selectedValue) ? selectedValue : supported[0];
backend/src/workspaces/contracts.ts
+1 -1
View File
@@ -131,7 +131,7 @@ export function buildInstallationContract(workspace: WorkspaceDescriptor): Insta
...connectorVariables(
namespace,
"VECTOR",
canonical.semantic_index.vector_store.supported_transports,
canonical.semantic_index.vector_store.supported_transports ?? [],
),
...(canonical.semantic_index.vector_writer
? [createVariable(namespace, "VECTOR_WRITER", "API_KEY_FILE")]
backend/src/workspaces/migrate-v2-qdrant.ts
+28
View File
@@ -0,0 +1,28 @@
import { validateOperationalWorkspace, type WorkspaceV2, type WorkspaceV3 } from "./schema.js";
export function migrateWorkspaceV2ToV3(
legacy: WorkspaceV2,
collection: string,
): WorkspaceV3 {
return validateOperationalWorkspace({
workspace: { ...legacy.workspace, schema_version: 3 },
dwh: legacy.dwh,
semantic_index: {
vector_store: {
engine: "qdrant",
collection,
dimensions: 1024,
distance: "cosine",
},
embedding: {
provider: "ollama_internal",
model: "qwen3-embedding:0.6b",
dimensions: 1024,
},
},
llm_policy: legacy.llm_policy,
...(legacy.diagnostics?.dwh_rest
? { diagnostics: { dwh_rest: legacy.diagnostics.dwh_rest } }
: {}),
});
}
backend/src/workspaces/runtime-renderer.ts
+2 -2
View File
@@ -99,8 +99,8 @@ export function renderRuntimeConfig(
};
const dwhIdentity = { database: canonical.dwh.database, schema: canonical.dwh.schema };
const vectorIdentity = {
database: canonical.semantic_index.vector_store.database,
schema: canonical.semantic_index.vector_store.schema,
database: canonical.semantic_index.vector_store.database ?? canonical.dwh.database,
schema: canonical.semantic_index.vector_store.schema ?? canonical.dwh.schema,
};
const dwhDirect = bindings.dwh.transport === "postgres_direct";
const vectorDirect = bindings.vector.transport === "pgvector_direct";
backend/src/workspaces/schema.ts
+94 -25
View File
@@ -34,7 +34,7 @@ export interface CanonicalDiagnostics {
embedding?: RestDiagnosticRequest & { response: { model: string; dimensions: string } };
}
interface WorkspaceMetadata<Version extends 1 | 2> {
interface WorkspaceMetadata<Version extends 1 | 2 | 3> {
schema_version: Version;
id: string;
name: string;
@@ -72,23 +72,52 @@ interface SemanticIndex<TVectorStore extends VectorStore> {
};
}
interface WorkspaceBase<Version extends 1 | 2, TVectorStore extends VectorStore> {
interface WorkspaceBase<Version extends 1 | 2 | 3, TVectorStore, TDiagnostics = CanonicalDiagnostics> {
workspace: WorkspaceMetadata<Version>;
dwh: WorkspaceDwh;
semantic_index: SemanticIndex<TVectorStore>;
semantic_index: {
vector_store: TVectorStore;
vector_writer?: Record<string, never>;
embedding: Version extends 3 ? {
provider: "ollama_internal";
model: "qwen3-embedding:0.6b";
dimensions: 1024;
timeout_ms?: number;
} : {
provider: "ollama_compatible" | "openai_compatible";
model: string;
dimensions: number;
timeout_ms?: number;
};
};
llm_policy: {
default?: `${string}/${string}`;
allowed: `${string}/${string}`[];
};
diagnostics?: CanonicalDiagnostics;
diagnostics?: TDiagnostics;
}
export interface CanonicalWorkspace extends WorkspaceBase<2, VectorStore & { database: string; schema: string }> {}
interface QdrantVectorStore {
engine: "qdrant";
collection: string;
dimensions: 1024;
distance: "cosine";
database?: string;
schema?: string;
port?: number;
timeout_ms?: number;
supported_transports?: VectorTransport[];
}
export interface WorkspaceV3 extends WorkspaceBase<3, QdrantVectorStore> {}
export interface WorkspaceV2 extends WorkspaceBase<2, VectorStore & { database: string; schema: string }> {}
/** A readable, non-operational v1 descriptor. It must be explicitly migrated before use. */
export interface LegacyWorkspace extends WorkspaceBase<1, VectorStore & { database?: string; schema?: string }> {}
export interface WorkspaceV1 extends WorkspaceBase<1, VectorStore & { database?: string; schema?: string }> {}
export type WorkspaceDescriptor = CanonicalWorkspace | LegacyWorkspace;
export type CanonicalWorkspace = WorkspaceV3;
export type LegacyWorkspace = WorkspaceV1 | WorkspaceV2;
export type WorkspaceDescriptor = WorkspaceV1 | WorkspaceV2 | WorkspaceV3;
const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, {
message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$",
@@ -159,6 +188,11 @@ const embeddingSchema = z.object({
dimensions,
timeout_ms: timeoutMs.optional(),
}).strict();
const internalEmbeddingSchema = z.object({
provider: z.literal("ollama_internal"),
model: z.literal("qwen3-embedding:0.6b"),
dimensions: z.literal(1024),
}).strict();
const vectorStoreShape = {
engine: z.literal("pgvector"),
collection: identifier,
@@ -178,6 +212,12 @@ const canonicalVectorStoreSchema = z.object({
database: identifier,
schema: identifier,
}).strict();
const qdrantVectorStoreSchema = z.object({
engine: z.literal("qdrant"),
collection: workspaceId,
dimensions: z.literal(1024),
distance: z.literal("cosine"),
}).strict();
const llmPolicySchema = z.object({
default: modelReference.optional(),
allowed: z.array(modelReference).min(1),
@@ -191,11 +231,13 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert
function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
unique(
workspace.semantic_index.vector_store.supported_transports,
context,
["semantic_index", "vector_store", "supported_transports"],
);
if ("supported_transports" in workspace.semantic_index.vector_store) {
unique(
workspace.semantic_index.vector_store.supported_transports,
context,
["semantic_index", "vector_store", "supported_transports"],
);
}
unique(workspace.llm_policy.allowed, context, ["llm_policy", "allowed"]);
if (workspace.semantic_index.vector_store.dimensions !== workspace.semantic_index.embedding.dimensions) {
@@ -221,7 +263,10 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
}
if (
workspace.diagnostics?.vector_rest
&& !workspace.semantic_index.vector_store.supported_transports.includes("rest_api")
&& (
!("supported_transports" in workspace.semantic_index.vector_store)
|| !workspace.semantic_index.vector_store.supported_transports.includes("rest_api")
)
) {
context.addIssue({
code: "custom",
@@ -236,7 +281,7 @@ const workspaceShape = {
llm_policy: llmPolicySchema,
diagnostics: diagnosticsSchema,
};
const LegacyWorkspaceSchema = z.object({
const WorkspaceV1Schema = z.object({
...workspaceShape,
workspace: z.object({
schema_version: z.literal(1), id: workspaceId, name: z.string().trim().min(1),
@@ -248,7 +293,7 @@ const LegacyWorkspaceSchema = z.object({
embedding: embeddingSchema,
}).strict(),
}).strict().superRefine(workspaceInvariants);
const CanonicalWorkspaceSchema = z.object({
const WorkspaceV2Schema = z.object({
...workspaceShape,
workspace: z.object({
schema_version: z.literal(2), id: workspaceId, name: z.string().trim().min(1),
@@ -260,7 +305,22 @@ const CanonicalWorkspaceSchema = z.object({
embedding: embeddingSchema,
}).strict(),
}).strict().superRefine(workspaceInvariants);
const WorkspaceDescriptorSchema = z.union([CanonicalWorkspaceSchema, LegacyWorkspaceSchema]);
const WorkspaceV3Schema = z.object({
dwh: dwhSchema,
llm_policy: llmPolicySchema,
diagnostics: z.object({
dwh_rest: dwhRestDiagnostic.optional(),
}).strict().optional(),
workspace: z.object({
schema_version: z.literal(3), id: workspaceId, name: z.string().trim().min(1),
description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]),
}).strict(),
semantic_index: z.object({
vector_store: qdrantVectorStoreSchema,
embedding: internalEmbeddingSchema,
}).strict(),
}).strict().superRefine(workspaceInvariants);
const WorkspaceDescriptorSchema = z.union([WorkspaceV3Schema, WorkspaceV2Schema, WorkspaceV1Schema]);
export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
const documents = parseAllDocuments(source, { uniqueKeys: true });
@@ -278,14 +338,23 @@ export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescri
}
export function isCanonicalWorkspace(workspace: WorkspaceDescriptor): workspace is CanonicalWorkspace {
return workspace.workspace.schema_version === 2;
return workspace.workspace.schema_version === 3;
}
export function isOperationalWorkspace(workspace: WorkspaceDescriptor): workspace is WorkspaceV3 {
return workspace.workspace.schema_version === 3;
}
/** Rejects readable v1 descriptors at every operational boundary until a caller migrates them. */
export function validateCanonicalWorkspace(workspace: unknown): CanonicalWorkspace {
return validateOperationalWorkspace(workspace);
}
/** Rejects readable v1/v2 descriptors at every operational boundary until a caller migrates them. */
export function validateOperationalWorkspace(workspace: unknown): WorkspaceV3 {
const descriptor = validateWorkspaceDescriptor(workspace);
if (!isCanonicalWorkspace(descriptor)) {
throw new Error("Workspace descriptor requires explicit migration to schema version 2");
if (!isOperationalWorkspace(descriptor)) {
throw new Error("Workspace descriptor requires explicit migration to schema version 3");
}
return descriptor;
}
@@ -295,19 +364,19 @@ export function validateCanonicalWorkspace(workspace: unknown): CanonicalWorkspa
* transformer never derives it from DWH identity, even where both services share a database.
*/
export function migrateWorkspaceV1ToV2(
workspace: LegacyWorkspace,
workspace: WorkspaceV1,
vectorIdentity: { database: string; schema: string },
): CanonicalWorkspace {
const legacy = LegacyWorkspaceSchema.parse(workspace) as LegacyWorkspace;
): WorkspaceV2 {
const legacy = WorkspaceV1Schema.parse(workspace) as WorkspaceV1;
const identity = z.object({ database: identifier, schema: identifier }).strict().parse(vectorIdentity);
return validateCanonicalWorkspace({
return WorkspaceV2Schema.parse({
...legacy,
workspace: { ...legacy.workspace, schema_version: 2 },
semantic_index: {
...legacy.semantic_index,
vector_store: { ...legacy.semantic_index.vector_store, ...identity },
},
});
}) as WorkspaceV2;
}
/** Builds a request URL only after rejecting values that can leave the declared service origin. */
@@ -320,7 +389,7 @@ export function resolveDiagnosticUrl(baseUrl: string, path: string): URL {
}
export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string {
const canonical = validateCanonicalWorkspace(workspace);
const canonical = validateOperationalWorkspace(workspace);
return stringify(canonical, { lineWidth: 0, sortMapEntries: true });
}
backend/src/workspaces/types.ts
+100
View File
@@ -15,3 +15,103 @@ export type WorkspaceErrorCode =
| "workspace_stale" | "workspace_conflict" | "git_unavailable"
| "git_auth_failed" | "git_non_fast_forward" | "git_push_rejected"
| "connector_unavailable" | "semantic_index_incompatible";
export interface QdrantVectorStore {
engine: "qdrant";
collection: string;
dimensions: 1024;
distance: "cosine";
database?: string;
schema?: string;
port?: number;
timeout_ms?: number;
supported_transports?: ("pgvector_direct" | "rest_api" | "ssh_tunnel")[];
}
export interface InternalEmbedding {
provider: "ollama_internal";
model: "qwen3-embedding:0.6b";
dimensions: 1024;
timeout_ms?: number;
}
export interface WorkspaceV2 {
workspace: {
schema_version: 2;
id: string;
name: string;
description?: string;
language: "en" | "it";
};
dwh: {
engine: "postgres";
database: string;
schema: string;
port?: number;
timeout_ms?: number;
supported_transports: ("postgres_direct" | "rest_api" | "ssh_tunnel")[];
};
semantic_index: {
vector_store: {
engine: "pgvector";
database: string;
schema: string;
collection: string;
dimensions: number;
distance: "cosine" | "l2" | "inner_product";
port?: number;
timeout_ms?: number;
supported_transports: ("pgvector_direct" | "rest_api" | "ssh_tunnel")[];
};
vector_writer?: Record<string, never>;
embedding: {
provider: "ollama_compatible" | "openai_compatible";
model: string;
dimensions: number;
timeout_ms?: number;
};
};
llm_policy: {
default?: `${string}/${string}`;
allowed: `${string}/${string}`[];
};
diagnostics?: {
dwh_rest?: {
method: "GET" | "POST";
path: string;
auth: "none" | "bearer" | "x-api-key";
response: { database: string; schema: string };
};
vector_rest?: {
metadata: {
method: "GET" | "POST";
path: string;
auth: "none" | "bearer" | "x-api-key";
response: { collection: string; dimensions: string; distance: string };
};
reversible_probe?: {
method: "POST";
path: string;
auth: "bearer" | "x-api-key";
response: { operation: string };
};
};
embedding?: {
method: "GET" | "POST";
path: string;
auth: "none" | "bearer" | "x-api-key";
response: { model: string; dimensions: string };
};
};
}
export interface WorkspaceV3 {
workspace: WorkspaceV2["workspace"] & { schema_version: 3 };
dwh: WorkspaceV2["dwh"];
semantic_index: {
vector_store: QdrantVectorStore;
embedding: InternalEmbedding;
};
llm_policy: WorkspaceV2["llm_policy"];
diagnostics?: WorkspaceV2["diagnostics"];
}