feat: unify installation model catalog

2026-09-02 18:45:33 +02:00
parent ae053961a3
commit 7b7927bfe5
169 changed files with 3696 additions and 4572 deletions
@@ -59,12 +59,12 @@ function safeSecretFilePath(path: string, secretRoots: readonly string[]): strin

 function requireSupportedDescriptor(workspace: unknown): void {
  if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Workspace bindings support only workspace schema version 3");
+    throw new Error("Workspace bindings support only workspace schema version 4");
  }
  const metadata = Reflect.get(workspace, "workspace");
  if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Workspace bindings support only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Workspace bindings support only workspace schema version 4");
  }
 }

@@ -155,7 +155,7 @@ export function resolveEvidenceBinding(
  return { values, missing };
 }

-/** Resolve the complete schema-v3 runtime binding set. */
+/** Resolve the complete schema-v4 runtime binding set. */
 export function resolveRuntimeBindings(
  workspace: WorkspaceDescriptor,
  env: NodeJS.ProcessEnv,
@@ -137,12 +137,12 @@ function evidenceVariables(

 function requireSupportedDescriptor(workspace: unknown): void {
  if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Installation contract supports only workspace schema version 3");
+    throw new Error("Installation contract supports only workspace schema version 4");
  }
  const metadata = Reflect.get(workspace, "workspace");
  if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Installation contract supports only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Installation contract supports only workspace schema version 4");
  }
 }

@@ -406,12 +406,12 @@ function numericBinding(binding: Record<string, string>, name: string): number |

 function requireSupportedDescriptor(workspace: unknown): void {
  if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Workspace diagnoser supports only workspace schema version 3");
+    throw new Error("Workspace diagnoser supports only workspace schema version 4");
  }
  const metadata = Reflect.get(workspace, "workspace");
  if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Workspace diagnoser supports only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Workspace diagnoser supports only workspace schema version 4");
  }
 }

@@ -506,11 +506,15 @@ async function diagnoseValidatedWorkspace(
  try {
    const vector = await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
      baseUrl: semanticRuntime.internalQdrantUrl,
-      collection: descriptor.semantic_index.vector_store.collection,
+      collection: descriptor.workspace.id,
      timeoutMs,
      signal,
    }));
-    const expected = descriptor.semantic_index.vector_store;
+    const expected = {
+      collection: descriptor.workspace.id,
+      dimensions: semanticRuntime.internalEmbeddingDimensions,
+      distance: "cosine",
+    };
    if (vector.collection !== expected.collection
      || vector.dimensions !== expected.dimensions
      || vector.distance !== expected.distance) {
@@ -529,10 +533,8 @@ async function diagnoseValidatedWorkspace(
      timeoutMs,
      signal,
    }));
-    if (semanticRuntime.internalEmbeddingModel !== descriptor.semantic_index.embedding.model
-      || semanticRuntime.internalEmbeddingDimensions !== descriptor.semantic_index.embedding.dimensions
-      || !embedding.available
-      || embedding.dimensions !== descriptor.semantic_index.embedding.dimensions) {
+    if (!embedding.available
+      || embedding.dimensions !== semanticRuntime.internalEmbeddingDimensions) {
      diagnostics.push(diagnosticError("semantic_index_incompatible"));
      activatable = false;
    }
@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
 import { normalize } from "node:path";

 export interface CanonicalEffectiveConfig {
-  schemaVersion: 1;
+  schemaVersion: 2;
  dwh: CanonicalDwhConfig;
  vector: CanonicalVectorConfig;
  embedding: CanonicalEmbeddingConfig;
@@ -27,6 +27,7 @@ export interface CanonicalVectorConfig {
 }

 export interface CanonicalEmbeddingConfig {
+  id: string;
  model: string;
  dimensions: number;
 }
@@ -137,8 +138,10 @@ function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbed
  if (!embeddings) {
    throw new TypeError("effective config is missing embedding resources");
  }
+  const model = requireString(embeddings, "model");
  return {
-    model: requireString(embeddings, "model"),
+    id: `ollama/${model}`,
+    model,
    dimensions: requireNumber(embeddings, "dimensions"),
  };
 }
@@ -166,7 +169,7 @@ export function buildCanonicalEffectiveConfig(renderedConfig: unknown): Canonica
    throw new TypeError("effective config requires a rendered configuration object");
  }
  return {
-    schemaVersion: 1,
+    schemaVersion: 2,
    dwh: buildDwhConfig(rendered),
    vector: buildVectorConfig(rendered),
    embedding: buildEmbeddingConfig(rendered),
@@ -77,6 +77,7 @@ export interface WorkspacePreprocessingServiceDeps {
    { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
  >;
  httpPrivateHostAllowlist?: readonly string[];
+  embeddingDimensions?: number;
 }

 interface RunScope {
@@ -118,7 +119,7 @@ export class WorkspacePreprocessingService {

  async vectorInspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-    const collection = runtime.workspace.semantic_index.vector_store.collection;
+    const collection = runtime.workspace.workspace.id;
    const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" });
    if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] });
    const body = await res.json() as any;
@@ -132,7 +133,7 @@ export class WorkspacePreprocessingService {

  async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise<WorkspaceOperationResult> {
    const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
-    const collection = runtime.workspace.semantic_index.vector_store.collection;
+    const collection = runtime.workspace.workspace.id;
    if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) {
      return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] });
    }
@@ -143,8 +144,8 @@ export class WorkspacePreprocessingService {
    const recreated = await reconcileCollection({
      baseUrl: runtime.configLease.semanticQdrantUrl,
      collection,
-      dimensions: runtime.workspace.semantic_index.vector_store.dimensions,
-      distance: runtime.workspace.semantic_index.vector_store.distance,
+      dimensions: this.deps.embeddingDimensions ?? 1024,
+      distance: "cosine",
      mode: "self_heal",
    });
    if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] });
@@ -400,6 +401,8 @@ export class WorkspacePreprocessingService {
      catalogBlob: runtime.catalogBlob,
      configDigest: runtime.configLease.configDigest,
      bindingDigest: runtime.configLease.bindingDigest,
+      embeddingId: runtime.configLease.effectiveConfig.embedding.id,
+      embeddingDimensions: runtime.configLease.effectiveConfig.embedding.dimensions,
    });
    return { runtime, state, job };
  }
@@ -44,11 +44,13 @@ export interface BeginPreprocessingJobOptions {
  catalogBlob: string;
  configDigest: string;
  bindingDigest: string;
+  embeddingId: string;
+  embeddingDimensions: number;
  runId?: string;
 }

 export interface PreprocessingJobState {
-  schemaVersion: 1;
+  schemaVersion: 2;
  runId: string;
  operation: string;
  workspaceId: string;
@@ -57,6 +59,8 @@ export interface PreprocessingJobState {
  catalogBlob: string;
  configDigest: string;
  bindingDigest: string;
+  embeddingId: string;
+  embeddingDimensions: number;
  completedStages: string[];
  childRuns: Record<string, string>;
  status: "active" | "succeeded" | "blocked" | "failed";
@@ -146,7 +150,7 @@ function decodeJob(value: unknown): PreprocessingJobState {
  }
  const record = value as Record<string, unknown>;
  if (
-    record.schemaVersion !== 1
+    record.schemaVersion !== 2
    || typeof record.runId !== "string"
    || typeof record.operation !== "string"
    || typeof record.workspaceId !== "string"
@@ -155,6 +159,8 @@ function decodeJob(value: unknown): PreprocessingJobState {
    || typeof record.catalogBlob !== "string"
    || typeof record.configDigest !== "string"
    || typeof record.bindingDigest !== "string"
+    || typeof record.embeddingId !== "string"
+    || typeof record.embeddingDimensions !== "number"
    || !Array.isArray(record.completedStages)
    || typeof record.childRuns !== "object" || record.childRuns === null || Array.isArray(record.childRuns)
    || !["active", "succeeded", "blocked", "failed"].includes(String(record.status))
@@ -275,6 +281,8 @@ export class PreprocessingStateStore {
        || existing.catalogBlob !== options.catalogBlob
        || existing.configDigest !== options.configDigest
        || existing.bindingDigest !== options.bindingDigest
+        || existing.embeddingId !== options.embeddingId
+        || existing.embeddingDimensions !== options.embeddingDimensions
      ) {
        throw new PreprocessingStateError(
          "preprocessing_resume_mismatch",
@@ -295,7 +303,7 @@ export class PreprocessingStateStore {
      }
    }
    const job: PreprocessingJobState = {
-      schemaVersion: 1,
+      schemaVersion: 2,
      runId,
      operation: options.operation,
      workspaceId: this.options.workspaceId,
@@ -304,6 +312,8 @@ export class PreprocessingStateStore {
      catalogBlob: options.catalogBlob,
      configDigest: options.configDigest,
      bindingDigest: options.bindingDigest,
+      embeddingId: options.embeddingId,
+      embeddingDimensions: options.embeddingDimensions,
      completedStages: [],
      childRuns: {},
      status: "active",
@@ -560,7 +560,7 @@ export class WorkspaceRegistry {
            });
          }
        }
-        const collection = workspace.semantic_index.vector_store.collection;
+        const collection = workspace.workspace.id;
        const owner = collectionOwners.get(collection);
        if (owner !== undefined) {
          throw new Error(`duplicate qdrant collection ownership: ${collection} (${owner}, ${id})`);
@@ -34,6 +34,7 @@ export interface RuntimeInstallationOverlay {
 export interface SemanticRuntimeConfig {
  internalQdrantUrl: string;
  internalEmbeddingUrl: string;
+  internalEmbeddingId?: string;
  internalEmbeddingModel: string;
  internalEmbeddingDimensions: number;
 }
@@ -41,6 +42,7 @@ export interface SemanticRuntimeConfig {
 export const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
  internalQdrantUrl: "http://qdrant:6333",
  internalEmbeddingUrl: "http://embedding:11434",
+  internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
  internalEmbeddingModel: "qwen3-embedding:0.6b",
  internalEmbeddingDimensions: 1024,
 };
@@ -202,16 +204,16 @@ function placeholderConnection(identity: { database: string; schema: string }):

 function requireSupportedDescriptor(workspace: unknown): void {
  if (typeof workspace !== "object" || workspace === null) {
-    throw new Error("Runtime renderer supports only workspace schema version 3");
+    throw new Error("Runtime renderer supports only workspace schema version 4");
  }
  const metadata = Reflect.get(workspace, "workspace");
  if (typeof metadata !== "object" || metadata === null
-    || Reflect.get(metadata, "schema_version") !== 3) {
-    throw new Error("Runtime renderer supports only workspace schema version 3");
+    || Reflect.get(metadata, "schema_version") !== 4) {
+    throw new Error("Runtime renderer supports only workspace schema version 4");
  }
 }

-/** Render the schema-v3 compatibility fields consumed by the current Python harness. */
+/** Render the schema-v4 compatibility fields consumed by the current Python harness. */
 export function renderRuntimeConfig(
  workspace: WorkspaceDescriptor,
  bindings: RuntimeBindings,
@@ -263,12 +265,24 @@ export function renderRuntimeConfig(
    ...(installation.profile === undefined ? {} : { profile: installation.profile }),
    language: descriptor.workspace.language,
    database,
-    semantic_index: descriptor.semantic_index,
+    semantic_index: {
+      vector_store: {
+        engine: "qdrant",
+        collection: descriptor.workspace.id,
+        dimensions: semanticRuntime.internalEmbeddingDimensions,
+        distance: "cosine",
+      },
+      embedding: {
+        provider: "ollama_internal",
+        model: semanticRuntime.internalEmbeddingModel,
+        dimensions: semanticRuntime.internalEmbeddingDimensions,
+      },
+    },
    resources: {
      vector: {
        engine: "qdrant",
        base_url: semanticRuntime.internalQdrantUrl,
-        collection: descriptor.semantic_index.vector_store.collection,
+        collection: descriptor.workspace.id,
      },
      embeddings: {
        provider: "ollama_internal",
@@ -35,7 +35,7 @@ export interface CanonicalDiagnostics {
 }

 interface WorkspaceMetadata {
-  schema_version: 3;
+  schema_version: 4;
  id: string;
  name: string;
  description?: string;
@@ -51,31 +51,12 @@ interface WorkspaceDwh {
  supported_transports: DwhTransport[];
 }

-interface WorkspaceBase<TVectorStore> {
+interface WorkspaceBase {
  workspace: WorkspaceMetadata;
  dwh: WorkspaceDwh;
-  semantic_index: {
-    vector_store: TVectorStore;
-    embedding: {
-      provider: "ollama_internal";
-      model: "qwen3-embedding:0.6b";
-      dimensions: 1024;
-    };
-  };
-  llm_policy: {
-    default?: `${string}/${string}`;
-    allowed: `${string}/${string}`[];
-  };
  diagnostics?: Pick<CanonicalDiagnostics, "dwh_rest">;
 }

-interface QdrantVectorStore {
-  engine: "qdrant";
-  collection: string;
-  dimensions: 1024;
-  distance: "cosine";
-}
-
 export interface EvidencePolicy {
  max_chunk_chars: number;
  retain_published_generations: number;
@@ -120,12 +101,12 @@ export interface WorkspaceEvidence {
  policy: EvidencePolicy;
 }

-export interface WorkspaceV3 extends WorkspaceBase<QdrantVectorStore> {
+export interface WorkspaceV4 extends WorkspaceBase {
  evidence?: WorkspaceEvidence;
 }

-export type CanonicalWorkspace = WorkspaceV3;
-export type WorkspaceDescriptor = WorkspaceV3;
+export type CanonicalWorkspace = WorkspaceV4;
+export type WorkspaceDescriptor = WorkspaceV4;

 const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, {
  message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$",
@@ -135,9 +116,6 @@ const identifier = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, {
 });
 const port = z.number().int().min(1).max(65_535);
 const timeoutMs = z.number().int().positive();
-const modelReference = z.string().regex(/^[^/\s]+\/[^/\s]+$/, {
-  message: "model must use provider/model syntax",
-});

 function isOriginRelativeDiagnosticPath(value: string): boolean {
  return /^\/(?!\/)[^\\\u0000-\u001F\u007F?#]*$/.test(value) && !/%5c/i.test(value);
@@ -166,21 +144,6 @@ const dwhSchema = z.object({
  timeout_ms: timeoutMs.optional(),
  supported_transports: z.array(z.enum(DWH_TRANSPORTS)).min(1),
 }).strict();
-const internalEmbeddingSchema = z.object({
-  provider: z.literal("ollama_internal"),
-  model: z.literal("qwen3-embedding:0.6b"),
-  dimensions: z.literal(1024),
-}).strict();
-const qdrantVectorStoreSchema = z.object({
-  engine: z.literal("qdrant"),
-  collection: workspaceId,
-  dimensions: z.literal(1024),
-  distance: z.literal("cosine"),
-}).strict();
-const llmPolicySchema = z.object({
-  default: modelReference.optional(),
-  allowed: z.array(modelReference).min(1),
-}).strict();

 const positiveSafeInteger = z.number().int().safe().positive();
 const nonnegativeSafeInteger = z.number().int().safe().nonnegative();
@@ -366,7 +329,6 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert

 function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
  unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
-  unique(workspace.llm_policy.allowed, context, ["llm_policy", "allowed"]);

  if (workspace.evidence?.source.type === "filesystem") {
    const expected = `${workspace.workspace.id}/evidence`;
@@ -379,20 +341,6 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
    }
  }

-  if (workspace.semantic_index.vector_store.dimensions !== workspace.semantic_index.embedding.dimensions) {
-    context.addIssue({
-      code: "custom",
-      path: ["semantic_index", "embedding", "dimensions"],
-      message: "embedding dimensions must match vector store dimensions",
-    });
-  }
-  if (workspace.llm_policy.default && !workspace.llm_policy.allowed.includes(workspace.llm_policy.default)) {
-    context.addIssue({
-      code: "custom",
-      path: ["llm_policy", "default"],
-      message: "LLM default must be included in the allowlist",
-    });
-  }
  if (workspace.diagnostics?.dwh_rest && !workspace.dwh.supported_transports.includes("rest_api")) {
    context.addIssue({
      code: "custom",
@@ -402,23 +350,18 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
  }
 }

-const WorkspaceV3Schema = z.object({
+const WorkspaceV4Schema = z.object({
  dwh: dwhSchema,
-  llm_policy: llmPolicySchema,
  evidence: workspaceEvidenceSchema.optional(),
  diagnostics: z.object({
    dwh_rest: dwhRestDiagnostic.optional(),
  }).strict().optional(),
  workspace: z.object({
-    schema_version: z.literal(3), id: workspaceId, name: z.string().trim().min(1),
+    schema_version: z.literal(4), id: workspaceId, name: z.string().trim().min(1),
    description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]),
  }).strict(),
-  semantic_index: z.object({
-    vector_store: qdrantVectorStoreSchema,
-    embedding: internalEmbeddingSchema,
-  }).strict(),
 }).strict().superRefine(workspaceInvariants);
-const WorkspaceDescriptorSchema = WorkspaceV3Schema;
+const WorkspaceDescriptorSchema = WorkspaceV4Schema;

 export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
  const documents = parseAllDocuments(source, { uniqueKeys: true });
@@ -431,6 +374,25 @@ export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
  return validateWorkspaceDescriptor(document.toJSON());
 }

+/** Deterministically removes the two installation-owned v3 blocks without altering workspace data. */
+export function migrateWorkspaceV3Yaml(source: string): string {
+  const documents = parseAllDocuments(source, { uniqueKeys: true });
+  if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
+  const document = documents[0];
+  if (document.errors.length > 0 || document.warnings.length > 0) {
+    throw new Error("Invalid workspace YAML");
+  }
+  const value = document.toJSON() as Record<string, unknown>;
+  const metadata = value.workspace as Record<string, unknown> | undefined;
+  if (!metadata || metadata.schema_version !== 3) {
+    throw new Error("Workspace migration requires schema version 3");
+  }
+  metadata.schema_version = 4;
+  delete value.semantic_index;
+  delete value.llm_policy;
+  return serializeWorkspaceYaml(validateWorkspaceDescriptor(value));
+}
+
 export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescriptor {
  return WorkspaceDescriptorSchema.parse(workspace) as WorkspaceDescriptor;
 }
@@ -439,11 +401,11 @@ export function isCanonicalWorkspace(workspace: unknown): workspace is Canonical
  return WorkspaceDescriptorSchema.safeParse(workspace).success;
 }

-export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV3 {
+export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV4 {
  return WorkspaceDescriptorSchema.safeParse(workspace).success;
 }

-export function validateOperationalWorkspace(workspace: unknown): WorkspaceV3 {
+export function validateOperationalWorkspace(workspace: unknown): WorkspaceV4 {
  return validateWorkspaceDescriptor(workspace);
 }

@@ -19,4 +19,4 @@ export type WorkspaceErrorCode =
  | "workspace_stale" | "git_unavailable" | "git_auth_failed" | "git_non_fast_forward"
  | "connector_unavailable" | "semantic_index_incompatible";

-export type { WorkspaceV3 } from "./schema.js";
+export type { WorkspaceV4 } from "./schema.js";