diff --git a/backend/src/app.ts b/backend/src/app.ts index 91fba883..63cdd288 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -268,6 +268,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc ); const listModels = deps?.listModels ?? createPiModelLister(config, { + modelCatalog: runtimeModelCatalog, warn: (detail) => app.log.warn( { component: "pi-model-list", detail }, "Pi enabled-model configuration warning", diff --git a/backend/src/catalog/migrations/010_canonical_model_ids.ts b/backend/src/catalog/migrations/010_canonical_model_ids.ts index 2cdfa8cd..147b0319 100644 --- a/backend/src/catalog/migrations/010_canonical_model_ids.ts +++ b/backend/src/catalog/migrations/010_canonical_model_ids.ts @@ -3,16 +3,17 @@ import type { CatalogDatabase } from "../repository.js"; const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$"; const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$"; +const historicalOrCanonicalModelPattern = `(${legacyModelPattern})|(${canonicalModelPattern})`; export async function up(db: Kysely): Promise { await sql.raw(`alter table description_generation_runs drop constraint description_generation_runs_model_id_check, add constraint description_generation_runs_model_id_check - check (model_id ~ '${canonicalModelPattern}')`).execute(db); + check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db); await sql.raw(`alter table sensitive_data_suggestion_runs drop constraint sensitive_data_suggestion_runs_model_id_check, add constraint sensitive_data_suggestion_runs_model_id_check - check (model_id ~ '${canonicalModelPattern}')`).execute(db); + check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db); } export async function down(db: Kysely): Promise { diff --git a/backend/src/pi/list-models.ts b/backend/src/pi/list-models.ts index ebdafa55..653f44e7 100644 --- a/backend/src/pi/list-models.ts +++ b/backend/src/pi/list-models.ts @@ -10,6 +10,7 @@ import { readConfiguredPiAgentFile, validateDeclarativePiConfig, } from "./managed-config.js"; +import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js"; export interface PiModel { provider: string; @@ -30,6 +31,7 @@ interface Opts { nowMs?: () => number; loadEnabledModels?: () => PiEnabledModelsResult; readModelsStore?: () => string | undefined; + modelCatalog?: RuntimeModelCatalog; warn?: (message: string) => void; } @@ -82,9 +84,23 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModels const byCompositeId = new Map( available.map((model) => [`${model.provider}/${model.id}`, model]), ); + const catalogByPiId = new Map( + (opts.modelCatalog?.sessionModels() ?? []).map((model) => [ + `${model.provider}/${model.upstreamModel}`, + model, + ]), + ); const models = enabled.ids.flatMap((id) => { const model = byCompositeId.get(id); - return model ? [model] : []; + if (!model) return []; + const catalogModel = catalogByPiId.get(id); + return [catalogModel ? { + ...model, + provider: catalogModel.provider, + id: catalogModel.model, + name: catalogModel.label, + reasoning: catalogModel.session?.reasoning ?? model.reasoning, + } : model]; }); if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available"); cache = { at: now(), models }; diff --git a/backend/src/pi/pi-process-manager.ts b/backend/src/pi/pi-process-manager.ts index 08da76de..ffc61927 100644 --- a/backend/src/pi/pi-process-manager.ts +++ b/backend/src/pi/pi-process-manager.ts @@ -260,8 +260,11 @@ export class PiProcessManager { const thinking = o.thinking ?? this.cfg.defaults.thinking; if (provider && model) { + const upstreamModel = this.modelCatalog.sessionModels() + .find((entry) => entry.provider === provider && entry.model === model) + ?.upstreamModel ?? model; const response = await rt.rpc.request( - { type: "set_model", provider, modelId: model } as object & { type: string }, + { type: "set_model", provider, modelId: upstreamModel } as object & { type: string }, ); rt.bridge.setContextWindow(response?.data?.contextWindow); } diff --git a/backend/src/pi/provider-smoke.ts b/backend/src/pi/provider-smoke.ts index 3def9147..88937054 100644 --- a/backend/src/pi/provider-smoke.ts +++ b/backend/src/pi/provider-smoke.ts @@ -79,6 +79,7 @@ export function createPiProviderSmoke( || catalog.defaultSession !== null; const catalogModel = catalog.sessionModels() .find((entry) => entry.provider === canonicalProvider && entry.model === model); + const upstreamModel = catalogModel?.upstreamModel ?? model; const credentialName = catalogModel?.authentication.mode === "secret_env" ? catalogModel.authentication.apiKeyEnv : undefined; @@ -130,7 +131,7 @@ export function createPiProviderSmoke( const capabilityGuard = failOnUnexpectedCapabilities(rpc); const turn = async (): Promise => { requireSuccessfulResponse(await rpc.request({ - type: "set_model", provider: canonicalProvider, modelId: model, + type: "set_model", provider: canonicalProvider, modelId: upstreamModel, } as object & { type: string })); requireSuccessfulResponse(await rpc.request({ type: "set_thinking_level", level: reasoning, diff --git a/backend/src/workspaces/effective-config.ts b/backend/src/workspaces/effective-config.ts index 031bb17e..02a6ccef 100644 --- a/backend/src/workspaces/effective-config.ts +++ b/backend/src/workspaces/effective-config.ts @@ -140,7 +140,7 @@ function buildEmbeddingConfig(rendered: Record): CanonicalEmbed } const model = requireString(embeddings, "model"); return { - id: `ollama/${model}`, + id: optionalString(embeddings, "id") ?? `ollama/${model}`, model, dimensions: requireNumber(embeddings, "dimensions"), }; diff --git a/backend/src/workspaces/runtime-renderer.ts b/backend/src/workspaces/runtime-renderer.ts index 0a9f4652..01a71be6 100644 --- a/backend/src/workspaces/runtime-renderer.ts +++ b/backend/src/workspaces/runtime-renderer.ts @@ -274,6 +274,8 @@ export function renderRuntimeConfig( }, embedding: { provider: "ollama_internal", + id: semanticRuntime.internalEmbeddingId + ?? `ollama/${semanticRuntime.internalEmbeddingModel}`, model: semanticRuntime.internalEmbeddingModel, dimensions: semanticRuntime.internalEmbeddingDimensions, }, @@ -287,6 +289,8 @@ export function renderRuntimeConfig( embeddings: { provider: "ollama_internal", base_url: semanticRuntime.internalEmbeddingUrl, + id: semanticRuntime.internalEmbeddingId + ?? `ollama/${semanticRuntime.internalEmbeddingModel}`, model: semanticRuntime.internalEmbeddingModel, dimensions: semanticRuntime.internalEmbeddingDimensions, }, diff --git a/backend/test/catalog-repository.integration.test.ts b/backend/test/catalog-repository.integration.test.ts index 533b26e6..91236a2d 100644 --- a/backend/test/catalog-repository.integration.test.ts +++ b/backend/test/catalog-repository.integration.test.ts @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { PostgreSqlContainer } from "@testcontainers/postgresql"; import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely"; import { Pool } from "pg"; @@ -33,7 +34,30 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo await upDescriptionGeneration(db); await upSensitiveSuggestionRuns(db); await upAiTokenUsage(db); + const historicalDatabaseId = randomUUID(); + await db.insertInto("workspaceDatabases").values({ + id: historicalDatabaseId, + workspaceId: "migration-history", + engine: "postgres", + databaseName: "warehouse", + schemaName: "public", + }).execute(); + await db.insertInto("descriptionGenerationRuns").values({ + id: randomUUID(), databaseId: historicalDatabaseId, scope: "all", + modelId: "openai-mini", language: "en", status: "completed", total: 1, + processed: 1, generated: 1, + }).execute(); + await db.insertInto("sensitiveDataSuggestionRuns").values({ + id: randomUUID(), databaseId: historicalDatabaseId, scope: "all", + modelId: "openai-mini", status: "completed", total: 1, + suggestedSensitive: 1, + }).execute(); await upCanonicalModelIds(db); + await expect(db.insertInto("descriptionGenerationRuns").values({ + id: randomUUID(), databaseId: historicalDatabaseId, scope: "all", + modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1, + processed: 1, generated: 1, + }).execute()).resolves.toBeDefined(); await sql`CREATE ROLE thothii_catalog_runtime`.execute(db); await upRuntimeSequencePrivileges(db); const sequencePrivilege = await sql<{ allowed: boolean }>` diff --git a/backend/test/list-models.test.ts b/backend/test/list-models.test.ts index 3f615a0c..ba1930eb 100644 --- a/backend/test/list-models.test.ts +++ b/backend/test/list-models.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path"; import path from "node:path"; import { createPiModelLister } from "../src/pi/list-models.js"; import { loadConfig } from "../src/config.js"; +import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js"; const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs"); @@ -44,6 +45,36 @@ test("createPiModelLister returns mapped PiModel[] from get_available_models", a } }); +test("catalog listing translates upstream Pi IDs back to canonical model keys", async () => { + const script = scriptWith([ + { provider: "local", id: "qwen2.5:7b", name: "Upstream label", reasoning: false }, + ]); + const model: RuntimeModel = { + id: "local/qwen", provider: "local", model: "qwen", label: "Catalog Qwen", + upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" }, + authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" }, + session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 }, + }; + const modelCatalog: RuntimeModelCatalog = { + defaultSession: model.id, defaultMetadataGeneration: null, embedding: null, + sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id, + }; + try { + const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + ...noManagedModels, + modelCatalog, + loadEnabledModels: enabled("local/qwen2.5:7b"), + spawnFn: () => spawn("node", [FAKE, script]) as any, + }); + + await expect(lister()).resolves.toEqual([{ + provider: "local", id: "qwen", name: "Catalog Qwen", reasoning: true, + }]); + } finally { + rmSync(path.dirname(script), { recursive: true, force: true }); + } +}); + test("createPiModelLister caches within ttl (spawns once for two calls)", async () => { const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]); try { diff --git a/backend/test/pi-process-manager.test.ts b/backend/test/pi-process-manager.test.ts index 6782be16..62cd9456 100644 --- a/backend/test/pi-process-manager.test.ts +++ b/backend/test/pi-process-manager.test.ts @@ -718,6 +718,54 @@ test.each([["OpenAI", "openai"], ["gemini", "google"]])( }, ); +test("set_model translates a canonical catalog key to its upstream Pi model ID", async () => { + const root = mkdtempSync(path.join(tmpdir(), "thothii-upstream-model-")); + const agentDir = path.join(root, "agent"); + mkdirSync(agentDir, { mode: 0o700 }); + writeFileSync(path.join(agentDir, "models.json"), JSON.stringify({ + providers: { + local: { + baseUrl: "http://ollama:11434/v1", apiKey: "local", + models: [{ id: "qwen2.5:7b" }], + }, + }, + }), { mode: 0o600 }); + vi.stubEnv("PI_CODING_AGENT_DIR", agentDir); + const child = recordingChild(); + child.stderr.resume = () => {}; + child.stdin.write = (data: unknown) => { + const request = JSON.parse(String(data)); + child._writes.push(String(data)); + if (request.id) { + queueMicrotask(() => child.stdout.emit("data", `${JSON.stringify({ + type: "response", id: request.id, success: true, + })}\n`)); + } + return true; + }; + const model: RuntimeModel = { + id: "local/qwen", provider: "local", model: "qwen", label: "Qwen", + upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" }, + authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" }, + session: { reasoning: false, contextWindow: 32768, maxTokens: 8192 }, + }; + const modelCatalog: RuntimeModelCatalog = { + defaultSession: model.id, defaultMetadataGeneration: null, embedding: null, + sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id, + }; + const mgr = new PiProcessManager(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), { + modelCatalog, authProviders: () => new Set(), spawnFn: () => child as any, + }); + try { + await mgr.spawnFor("upstream-model", { provider: "local", model: "qwen" }); + expect(child._writes.join("")).toContain('"modelId":"qwen2.5:7b"'); + } finally { + mgr.teardown("upstream-model"); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); + } +}); + test.each(["installation-local", "private-compatible"])( "provider %s configured with a literal apiKey spawns without a managed key", async (provider) => { diff --git a/backend/test/pi-provider-smoke.test.ts b/backend/test/pi-provider-smoke.test.ts index 3794fcda..3be1b525 100644 --- a/backend/test/pi-provider-smoke.test.ts +++ b/backend/test/pi-provider-smoke.test.ts @@ -300,11 +300,22 @@ test("provider smoke makes one configured request from an isolated no-capability }); } }); + const smokeModel: RuntimeModel = { + id: "zai/catalog-glm", provider: "zai", model: "catalog-glm", label: "GLM", + upstreamModel: "glm-5.2", authentication: { mode: "pi_auth" }, + sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true }, + }; + const smokeCatalog: RuntimeModelCatalog = { + defaultSession: smokeModel.id, defaultMetadataGeneration: null, embedding: null, + sessionModels: () => [smokeModel], metadataModels: () => [], + hasSession: (id) => id === smokeModel.id, + }; const smoke = createPiProviderSmoke(loadConfig({ THT_HARNESS_DIR: "/app/harness", PI_BIN: "/usr/local/bin/pi", THT_DATA_ROOT: "/mounted-workflow-state", }), { + modelCatalog: smokeCatalog, spawnFn: (...args) => { spawns.push(args); expect(args[2].cwd).not.toBe("/app/harness"); @@ -324,7 +335,7 @@ test("provider smoke makes one configured request from an isolated no-capability }); await expect(smoke({ - provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750, + provider: "zai", model: "catalog-glm", reasoning: "medium", timeoutMs: 750, })).resolves.toBeUndefined(); expect(spawns).toHaveLength(1); expect(spawns[0][0]).toBe("/usr/local/bin/pi"); diff --git a/backend/test/workspace-runtime-renderer.test.ts b/backend/test/workspace-runtime-renderer.test.ts index 2fa1107f..4c42a7ee 100644 --- a/backend/test/workspace-runtime-renderer.test.ts +++ b/backend/test/workspace-runtime-renderer.test.ts @@ -73,6 +73,7 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" }, embeddings: { provider: "ollama_internal", base_url: "http://embedding:11434", + id: "ollama/qwen3-embedding:0.6b", model: "qwen3-embedding:0.6b", dimensions: 1024, }, }, diff --git a/deploy/workspaces/server-sessions.yaml.example b/deploy/workspaces/server-sessions.yaml.example index 616c11fa..e099854e 100644 --- a/deploy/workspaces/server-sessions.yaml.example +++ b/deploy/workspaces/server-sessions.yaml.example @@ -35,7 +35,6 @@ resources: base_url: http://qdrant:6333 collection: server-sessions embeddings: + # id, model, and dimensions are injected from the Installation Model Catalog. provider: ollama_internal base_url: http://embedding:11434 - model: qwen3-embedding:0.6b - dimensions: 1024 diff --git a/docs/general/pi-configuration.md b/docs/general/pi-configuration.md index 2819d965..8aab896d 100644 --- a/docs/general/pi-configuration.md +++ b/docs/general/pi-configuration.md @@ -100,7 +100,7 @@ generated/ ├── pi/ │ ├── models.json │ └── settings.json -└── compose.model-catalog.yaml +└── compose.models.yaml ``` The normalized catalog is consumed by the backend. The Pi files and Compose override are boundary @@ -117,17 +117,19 @@ tht --installation "$INSTALLATION" start tht --installation "$INSTALLATION" doctor ``` -After editing `modelCatalog` or provider credentials, reload the current Pi image. Restart validates -the YAML and regenerates projections before recreating `core`: +After editing `modelCatalog` or provider credentials, apply the complete runtime projection with +the normal installation lifecycle, then run the Pi checks: ```bash -tht --installation "$INSTALLATION" pi restart --yes --drain +tht --installation "$INSTALLATION" start tht --installation "$INSTALLATION" pi doctor tht --installation "$INSTALLATION" pi test ``` -`tht pi update` changes the Pi version; it is not the configuration command. There is no -`tht pi configure` and no separate apply command. +`tht pi restart`, `tht pi update`, and `tht pi rollback` refuse to run while generated model +projections differ from `modelCatalog`: those commands recreate only `core`, so they must never +partially apply an embedding change. `tht pi update` changes the Pi version; it is not the +configuration command. There is no `tht pi configure` and no separate apply command. ## Migrating a legacy installation @@ -136,7 +138,7 @@ files, but never modifies them. Supply the facts that cannot be inferred safely candidate: ```bash -tht --installation /absolute/path/legacy-installation.yaml installation migrate \ +tht --installation /absolute/path/legacy/thothii-installation.yaml installation migrate \ --output /absolute/path/thothii-installation.v2.yaml \ --session-default zai/glm-5.3 \ --embedding-id ollama/qwen3-embedding:0.6b \ diff --git a/docs/plans/2026-09-02-installation-model-catalog.md b/docs/plans/2026-09-02-installation-model-catalog.md index f0898616..d30af71f 100644 --- a/docs/plans/2026-09-02-installation-model-catalog.md +++ b/docs/plans/2026-09-02-installation-model-catalog.md @@ -166,6 +166,8 @@ Generation is deterministic and published only after every candidate artifact va generation aborts start before Compose is invoked. `tht doctor` recomputes expected bytes and reports differences; no digest manifest or separate apply command exists. When projection bytes change, `tht start` recreates the affected services so they cannot continue with an older bind mount. +Pi-only restart, update, and rollback operations reject projection drift and direct the operator to +`tht start`, because applying only the core-facing files could leave embedding services stale. Generated projections are not backed up. Restore validates the canonical installation descriptor, regenerates every projection, and only then starts services. Base Compose files and `operator.env` diff --git a/harness/tests/test_config_resources.py b/harness/tests/test_config_resources.py index c9249d1f..1175fb77 100644 --- a/harness/tests/test_config_resources.py +++ b/harness/tests/test_config_resources.py @@ -214,7 +214,7 @@ embeddings: {provider: ollama_internal, base_url: http://embedding:11434, model: assert cfg.vectors.writer.api_key == "writer" -def test_accepts_only_internal_ollama_embedding_contract(tmp_path): +def test_accepts_catalog_selected_internal_ollama_embedding_contract(tmp_path): workspace = tmp_path / "workspace.yaml" workspace.write_text( """ @@ -225,8 +225,9 @@ resources: embeddings: provider: ollama_internal base_url: http://embedding:11434 - model: qwen3-embedding:0.6b - dimensions: 1024 + id: ollama/bge-m3 + model: bge-m3 + dimensions: 1536 """ ) @@ -234,8 +235,59 @@ resources: assert cfg.embeddings.provider == "ollama_internal" assert cfg.embeddings.base_url == "http://embedding:11434" - assert cfg.embeddings.model == "qwen3-embedding:0.6b" - assert cfg.embeddings.dim == 1024 + assert cfg.embeddings.id == "ollama/bge-m3" + assert cfg.embeddings.model == "bge-m3" + assert cfg.embeddings.dim == 1536 + + +def test_installation_embedding_projection_completes_model_free_runtime_source( + tmp_path, monkeypatch, +): + monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3") + monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3") + monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536") + workspace = tmp_path / "workspace.yaml" + workspace.write_text( + """ +dwh: + type: postgres_direct + connection: {database: analytics, schema: mart, user: reader, password: secret} +resources: + embeddings: + provider: ollama_internal + base_url: http://embedding:11434 +""" + ) + + cfg = load_config(workspace) + + assert cfg.embeddings.id == "ollama/bge-m3" + assert cfg.embeddings.model == "bge-m3" + assert cfg.embeddings.dim == 1536 + + +def test_workspace_embedding_values_cannot_override_installation_projection( + tmp_path, monkeypatch, +): + monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3") + monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3") + monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536") + workspace = tmp_path / "workspace.yaml" + workspace.write_text( + """ +dwh: + type: postgres_direct + connection: {database: analytics, schema: mart, user: reader, password: secret} +resources: + embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: workspace-owned-model +""" + ) + + with pytest.raises(ConfigError, match="proprietà dell'installazione|diverge"): + load_config(workspace) def test_accepts_internal_qdrant_resource_contract(tmp_path): diff --git a/harness/tests/test_corpus_models.py b/harness/tests/test_corpus_models.py index 97b38317..772adbd9 100644 --- a/harness/tests/test_corpus_models.py +++ b/harness/tests/test_corpus_models.py @@ -40,6 +40,7 @@ def test_manifest_contains_provenance_without_credentials(): manifest_id="manifest:abc", created_at=datetime(2026, 7, 12, tzinfo=UTC), pipeline_version="evidence-v1", + embedding_id="ollama/nomic-embed-text", embedding_model="nomic-embed-text", embedding_dimensions=768, documents=[document()], @@ -52,6 +53,7 @@ def test_manifest_contains_provenance_without_credentials(): assert "etag:abc" in payload assert "evidence-v1" in payload assert "nomic-embed-text" in payload + assert '"embedding_id":"ollama/nomic-embed-text"' in payload assert "api_key" not in payload @@ -93,6 +95,25 @@ def test_manifest_validates_embedding_compatibility_fields(): ) +def test_schema_v1_manifest_without_canonical_embedding_id_remains_readable(): + manifest = CorpusManifest.model_validate({ + "schema_version": 1, + "embedding_model": "legacy-model", + "embedding_dimensions": 768, + }) + + assert manifest.embedding_id is None + + +def test_schema_v2_embedding_generation_requires_canonical_id(): + with pytest.raises(ValidationError, match="embedding_id"): + CorpusManifest( + schema_version=2, + embedding_model="model-v2", + embedding_dimensions=768, + ) + + def test_canonical_metadata_rejects_secrets_and_non_json_values(): with pytest.raises(ValidationError, match="credential-like"): CanonicalDocument.model_validate( diff --git a/harness/tests/test_corpus_pipeline.py b/harness/tests/test_corpus_pipeline.py index a0323a6f..ce3b49f0 100644 --- a/harness/tests/test_corpus_pipeline.py +++ b/harness/tests/test_corpus_pipeline.py @@ -105,11 +105,13 @@ def item(name, fingerprint): ) -def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None, +def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", + embedding_id=None, policy=None, retain=3, candidate_evaluator=None): return CorpusPipeline( store=CorpusStore(tmp_path / "corpus"), sources=[source], embedder=embedder or Embedder(), vector_store=vectors or Vectors(), + embedding_id=embedding_id, embedding_model=model, embedding_dimensions=3, chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100), pipeline_version="evidence-v1", @@ -875,6 +877,24 @@ def test_model_or_chunk_policy_change_forces_full_rebuild(tmp_path): assert source.acquire_calls == ["fs:one"] +def test_canonical_embedding_id_change_forces_full_rebuild_and_is_persisted(tmp_path): + one = item("one", "a") + first = pipeline( + tmp_path, Source([(one, "hello")]), model="same-upstream", + embedding_id="ollama/catalog-a", + ).run() + assert first.manifest.embedding_id == "ollama/catalog-a" + + source = Source([(one, "hello")]) + changed = pipeline( + tmp_path, source, model="same-upstream", embedding_id="ollama/catalog-b", + ).run() + + assert changed.changed == ("fs:one",) + assert changed.manifest.embedding_id == "ollama/catalog-b" + assert source.acquire_calls == ["fs:one"] + + def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path): one = item("one", "a") good = pipeline(tmp_path, Source([(one, "old")])) diff --git a/harness/tests/test_effective_config_p3.py b/harness/tests/test_effective_config_p3.py index 7e61bc26..4405cf56 100644 --- a/harness/tests/test_effective_config_p3.py +++ b/harness/tests/test_effective_config_p3.py @@ -21,7 +21,8 @@ def _write_cfg(tmp_path, raw): return cfg -def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid", collection="psd", model="qwen3-embedding:0.6b"): +def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid", + collection="psd", model="qwen3-embedding:0.6b", dimensions=1024): return { "schemaVersion": 1, "workspace": {"schema_version": 3, "id": "psd", "name": "PSD", "language": "it"}, @@ -34,7 +35,10 @@ def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.inval "connection": {"host": "h", "port": 5432, "database": "warehouse", "schema": "dw", "user": "reader", "password": "secret"}, }, "vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": collection, "collection_lifecycle": "self_heal"}, - "embeddings": {"provider": "ollama_internal", "base_url": "http://embedding:11434", "model": model, "dimensions": 1024}, + "embeddings": { + "provider": "ollama_internal", "base_url": "http://embedding:11434", + "id": f"ollama/{model}", "model": model, "dimensions": dimensions, + }, "roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")}, "paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")}, } @@ -52,6 +56,16 @@ def test_canonical_json_is_deterministic_and_key_ordered(cfg): assert keys == ["schemaVersion", "dwh", "vector", "embedding", "roots"] +def test_catalog_embedding_identity_and_dimensions_drive_effective_config(tmp_path): + cfg = _write_cfg(tmp_path, _cfg(tmp_path, model="bge-m3", dimensions=1536)) + document = __import__("json").loads(canonical_effective_config_json(cfg)) + + assert document["embedding"] == { + "id": "ollama/bge-m3", "model": "bge-m3", "dimensions": 1536, + } + assert document["vector"]["dimensions"] == 1536 + + def test_canonical_excludes_credentials_and_evidence(cfg): doc = canonical_effective_config_json(cfg) assert "secret" not in doc diff --git a/harness/tests/test_evidence_facade_contract.py b/harness/tests/test_evidence_facade_contract.py index a4cd5cce..ccaa0184 100644 --- a/harness/tests/test_evidence_facade_contract.py +++ b/harness/tests/test_evidence_facade_contract.py @@ -149,6 +149,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk "sources": [object()], "embedder": object(), "vector_store": object(), + "embedding_id": "ollama/model", "embedding_model": "model", "embedding_dimensions": 3, "chunk_policy": object(), diff --git a/harness/tests/test_ollama_ensure.py b/harness/tests/test_ollama_ensure.py index 9b8da931..c410da14 100644 --- a/harness/tests/test_ollama_ensure.py +++ b/harness/tests/test_ollama_ensure.py @@ -10,6 +10,8 @@ from tht.config import EmbeddingsConfig def _cfg(**kw): + kw.setdefault("model", "nomic-embed-text-v2-moe") + kw.setdefault("dim", 768) emb = EmbeddingsConfig(base_url="http://localhost:11434", **kw) return SimpleNamespace(embeddings=emb) diff --git a/harness/tests/test_preprocess_cli.py b/harness/tests/test_preprocess_cli.py index e0ec24ba..8f90e6d9 100644 --- a/harness/tests/test_preprocess_cli.py +++ b/harness/tests/test_preprocess_cli.py @@ -284,6 +284,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat sources, embedder, vector_store, + embedding_id, embedding_model, embedding_dimensions, chunk_policy, @@ -293,6 +294,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat candidate_evaluator, ): calls["init"] = { + "embedding_id": embedding_id, "embedding_model": embedding_model, "embedding_dimensions": embedding_dimensions, "pipeline_version": pipeline_version, @@ -314,6 +316,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat command.run_from_config(config) + assert calls["init"]["embedding_id"] == "ollama/qwen3-embedding:0.6b" assert calls["init"]["sparse_language"] == "english" assert calls["init"]["candidate_evaluator"] is None assert calls["run_as_job"]["workspace_id"] == "psd-clinical" diff --git a/harness/tht/cli/preprocess_cmd.py b/harness/tht/cli/preprocess_cmd.py index ad3cd262..be40c365 100644 --- a/harness/tht/cli/preprocess_cmd.py +++ b/harness/tht/cli/preprocess_cmd.py @@ -187,6 +187,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence), embedder=embedder, vector_store=vector_store, + embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}", embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", @@ -222,6 +223,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False): pipeline = build_preprocessing_pipeline( store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence), embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True), + embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}", embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", diff --git a/harness/tht/config.py b/harness/tht/config.py index eaa32d02..fb2f7c93 100644 --- a/harness/tht/config.py +++ b/harness/tht/config.py @@ -60,8 +60,12 @@ def canonical_effective_config_document(cfg) -> dict: return { "schemaVersion": 1, "dwh": dwh, - "vector": {"collection": collection, "dimensions": 1024, "distance": "cosine"}, - "embedding": {"model": model, "dimensions": int(embed_dim)}, + "vector": {"collection": collection, "dimensions": int(embed_dim), "distance": "cosine"}, + "embedding": { + "id": getattr(embeddings, "id", None) or f"ollama/{model}", + "model": model, + "dimensions": int(embed_dim), + }, "roots": { "artifacts": str(getattr(cfg.paths, "artifacts", Path("artifacts"))), "indexes": str(getattr(cfg.paths, "indexes", Path("indexes"))), @@ -499,8 +503,9 @@ class EvidenceSourcesConfig(BaseModel): class EmbeddingsConfig(BaseModel): provider: str = "ollama_internal" base_url: str - model: str = "nomic-embed-text-v2-moe" - dim: int = Field(default=768, alias="dimensions") + id: str | None = None + model: str + dim: int = Field(alias="dimensions") batch_size: int = 16 timeout: int = 300 connect_timeout: int = 5 @@ -642,6 +647,7 @@ def load_config(path: Path) -> Config: if not isinstance(raw, dict): raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}") expanded = _resolve_secret_files(_resolve_evidence_secret_files(_expand_env(raw))) + _apply_installation_embedding_projection(expanded, path) _validate_internal_embedding_contract(expanded, path) _validate_internal_vector_contract(expanded, path) translated, used_legacy = translate_legacy_config(expanded) @@ -710,6 +716,56 @@ def load_config(path: Path) -> Config: return cfg +def _apply_installation_embedding_projection(raw: dict[str, Any], path: Path) -> None: + projected = { + "id": os.environ.get("THT_INTERNAL_EMBEDDING_ID"), + "model": os.environ.get("THT_INTERNAL_EMBEDDING_MODEL"), + "dimensions": os.environ.get("THT_INTERNAL_EMBEDDING_DIMENSIONS"), + } + if all(value is None for value in projected.values()): + return + if any(value is None for value in projected.values()): + raise ConfigError( + f"Configurazione non valida in {path}:\n" + "la proiezione embedding dell'installazione è incompleta" + ) + embedding_id = projected["id"] + model = projected["model"] + try: + dimensions = int(projected["dimensions"] or "") + except ValueError: + dimensions = 0 + if embedding_id != f"ollama/{model}" or dimensions <= 0: + raise ConfigError( + f"Configurazione non valida in {path}:\n" + "la proiezione embedding dell'installazione non è canonica" + ) + + sections: list[tuple[dict[str, Any], str]] = [] + embeddings = raw.get("embeddings") + if isinstance(embeddings, dict): + sections.append((embeddings, "dim")) + resources = raw.get("resources") + if isinstance(resources, dict) and isinstance(resources.get("embeddings"), dict): + sections.append((resources["embeddings"], "dimensions")) + for section, dimension_key in sections: + declared = { + "id": section.get("id"), + "model": section.get("model"), + "dimensions": section.get(dimension_key), + } + expected = {"id": embedding_id, "model": model, "dimensions": dimensions} + for key, value in declared.items(): + if value is not None and str(value) != str(expected[key]): + raise ConfigError( + f"Configurazione non valida in {path}:\n" + f"{key} è proprietà dell'installazione e diverge dalla proiezione attiva" + ) + section["id"] = embedding_id + section["model"] = model + section[dimension_key] = dimensions + + def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> None: resources = raw.get("resources") if not isinstance(resources, dict): @@ -720,9 +776,10 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No provider = embeddings.get("provider") model = embeddings.get("model") + embedding_id = embeddings.get("id") dimensions = embeddings.get("dimensions") base_url = embeddings.get("base_url") - allowed = {"provider", "base_url", "model", "dimensions"} + allowed = {"provider", "base_url", "id", "model", "dimensions"} unexpected = sorted(set(embeddings) - allowed) if unexpected: raise ConfigError( @@ -734,15 +791,24 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No f"Configurazione non valida in {path}:\n" "resources.embeddings.provider deve essere 'ollama_internal'" ) - if model != "qwen3-embedding:0.6b": + if not isinstance(model, str) or not model: raise ConfigError( f"Configurazione non valida in {path}:\n" - "resources.embeddings.model deve essere 'qwen3-embedding:0.6b'" + "resources.embeddings.model deve essere valorizzato" ) - if dimensions != 1024: + try: + parsed_dimensions = int(dimensions) + except (TypeError, ValueError): + parsed_dimensions = 0 + if isinstance(dimensions, bool) or parsed_dimensions <= 0: raise ConfigError( f"Configurazione non valida in {path}:\n" - "resources.embeddings.dimensions deve essere 1024" + "resources.embeddings.dimensions deve essere un intero positivo" + ) + if embedding_id is not None and embedding_id != f"ollama/{model}": + raise ConfigError( + f"Configurazione non valida in {path}:\n" + "resources.embeddings.id deve essere l'identità canonica ollama/" ) if not _is_allowed_internal_embedding_url(base_url): raise ConfigError( @@ -805,15 +871,20 @@ def _validate_active_embeddings_config( f"Configurazione non valida in {path}:\n" "embeddings.provider deve essere 'ollama_internal'" ) - if embeddings.model != "qwen3-embedding:0.6b": + if not embeddings.model: raise ConfigError( f"Configurazione non valida in {path}:\n" - "embeddings.model deve essere 'qwen3-embedding:0.6b'" + "embeddings.model deve essere valorizzato" ) - if embeddings.dim != 1024: + if embeddings.dim <= 0: raise ConfigError( f"Configurazione non valida in {path}:\n" - "embeddings.dim deve essere 1024" + "embeddings.dim deve essere un intero positivo" + ) + if embeddings.id is not None and embeddings.id != f"ollama/{embeddings.model}": + raise ConfigError( + f"Configurazione non valida in {path}:\n" + "embeddings.id deve essere l'identità canonica ollama/" ) if not _is_allowed_internal_embedding_url(embeddings.base_url): raise ConfigError( diff --git a/harness/tht/evidence/corpus/models.py b/harness/tht/evidence/corpus/models.py index 69779bc5..928f8f03 100644 --- a/harness/tht/evidence/corpus/models.py +++ b/harness/tht/evidence/corpus/models.py @@ -19,6 +19,9 @@ from tht.evidence.contracts import ( _NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$") _SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") _EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$") +_CANONICAL_MODEL_ID = re.compile( + r"^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" +) _EVIDENCE_METADATA_KEYS = frozenset({ "evidence_id", "evidence_kind", "purposes", "scope", "language", "provenance", }) @@ -37,6 +40,12 @@ def _validate_hash(value: str) -> str: return value +def _validate_canonical_model_id(value: str) -> str: + if not _CANONICAL_MODEL_ID.fullmatch(value): + raise ValueError("model identifier must use canonical provider/model form") + return value + + def _require_content_hash(content: str, content_hash: str) -> None: expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}" if content_hash != expected: @@ -146,6 +155,7 @@ class CorpusManifest(_WithMetadata): manifest_id: str | None = None created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) pipeline_version: str = Field(default="evidence-v1", min_length=1) + embedding_id: str | None = None embedding_model: str | None = None embedding_dimensions: int | None = Field(default=None, gt=0) vector_generation: str | None = None @@ -155,6 +165,9 @@ class CorpusManifest(_WithMetadata): _manifest_id = field_validator("manifest_id")( lambda value: _validate_namespaced_id(value) if value is not None else None ) + _embedding_id = field_validator("embedding_id")( + lambda value: _validate_canonical_model_id(value) if value is not None else None + ) _vector_generation = field_validator("vector_generation")( lambda value: _validate_namespaced_id(value) if value is not None else None ) @@ -164,6 +177,10 @@ class CorpusManifest(_WithMetadata): def validate_generation(self) -> "CorpusManifest": if (self.embedding_model is None) != (self.embedding_dimensions is None): raise ValueError("embedding_model and embedding_dimensions must be set together") + if self.embedding_id is not None and self.embedding_model is None: + raise ValueError("embedding_id requires embedding model and dimensions") + if self.schema_version >= 2 and self.embedding_model is not None and self.embedding_id is None: + raise ValueError("schema version 2 embedding generations require embedding_id") if self.vector_generation is not None and self.embedding_model is None: raise ValueError("vector_generation requires embedding model and dimension compatibility") diff --git a/harness/tht/evidence/corpus/pipeline.py b/harness/tht/evidence/corpus/pipeline.py index 0b5973f2..df83245a 100644 --- a/harness/tht/evidence/corpus/pipeline.py +++ b/harness/tht/evidence/corpus/pipeline.py @@ -125,6 +125,7 @@ class CorpusPipeline: def __init__( self, *, store: CorpusStore, sources: list[EvidenceSource], embedder, vector_store: VectorStore, embedding_model: str, embedding_dimensions: int, + embedding_id: str | None = None, chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3, workspace_id: str | None = None, sparse_language: str = "italian", candidate_evaluator: Callable[[CorpusManifest], object] | None = None, @@ -133,6 +134,7 @@ class CorpusPipeline: self.sources = sources self.embedder = embedder self.vector_store = vector_store + self.embedding_id = embedding_id or f"ollama/{embedding_model}" self.embedding_model = embedding_model self.embedding_dimensions = embedding_dimensions self.chunk_policy = chunk_policy @@ -284,6 +286,7 @@ class CorpusPipeline: source_by_id = {item.source_id: (source, item) for source, item in discovered} compatibility = _fingerprint({ "pipeline": self.pipeline_version, + "embedding_id": self.embedding_id, "model": self.embedding_model, "dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy), @@ -294,6 +297,7 @@ class CorpusPipeline: "compatibility_fingerprint": compatibility, "pipeline_version": self.pipeline_version, "chunk_policy_version": self.chunk_policy.version, + "embedding_id": self.embedding_id, "embedding_model": self.embedding_model, "embedding_dimensions": self.embedding_dimensions, } @@ -492,7 +496,9 @@ class CorpusPipeline: ) for document in documents } manifest = CorpusManifest( + schema_version=2, pipeline_version=self.pipeline_version, + embedding_id=self.embedding_id, embedding_model=self.embedding_model, embedding_dimensions=self.embedding_dimensions, vector_generation=plan["generation"], @@ -723,7 +729,8 @@ class CorpusPipeline: prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {} fingerprints = {item.source_id: item.fingerprint for _, item in discovered} compatibility = _fingerprint({ - "pipeline": self.pipeline_version, "model": self.embedding_model, + "pipeline": self.pipeline_version, "embedding_id": self.embedding_id, + "model": self.embedding_model, "dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy), }) previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None @@ -761,7 +768,9 @@ class CorpusPipeline: for document in documents } manifest = CorpusManifest( + schema_version=2, pipeline_version=self.pipeline_version, + embedding_id=self.embedding_id, embedding_model=self.embedding_model, embedding_dimensions=self.embedding_dimensions, vector_generation=generation, diff --git a/harness/tht/evidence/preprocessing.py b/harness/tht/evidence/preprocessing.py index aff85696..7db67d9d 100644 --- a/harness/tht/evidence/preprocessing.py +++ b/harness/tht/evidence/preprocessing.py @@ -22,6 +22,7 @@ def build_preprocessing_pipeline( vector_store: VectorStore, embedding_model: str, embedding_dimensions: int, + embedding_id: str | None = None, chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3, @@ -35,6 +36,7 @@ def build_preprocessing_pipeline( sources=sources, embedder=embedder, vector_store=vector_store, + embedding_id=embedding_id, embedding_model=embedding_model, embedding_dimensions=embedding_dimensions, chunk_policy=chunk_policy, diff --git a/harness/workspaces/local.yaml b/harness/workspaces/local.yaml index 9a0753a6..64b67ba6 100644 --- a/harness/workspaces/local.yaml +++ b/harness/workspaces/local.yaml @@ -39,8 +39,6 @@ evidence: embeddings: base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434 - model: nomic-embed-text-v2-moe - dim: 768 batch_size: 32 # Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che diff --git a/harness/workspaces/tht-test.yaml b/harness/workspaces/tht-test.yaml index 7efb2b48..0967847b 100644 --- a/harness/workspaces/tht-test.yaml +++ b/harness/workspaces/tht-test.yaml @@ -25,8 +25,6 @@ paths: embeddings: base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 batch_size: 64 # LOADING diretto del pgvector (server-only). Su workstation la lettura passa da diff --git a/harness/workspaces/tht.example.yaml b/harness/workspaces/tht.example.yaml index 1a3438fe..18ece353 100644 --- a/harness/workspaces/tht.example.yaml +++ b/harness/workspaces/tht.example.yaml @@ -39,8 +39,6 @@ evidence: embeddings: base_url: ${THT_OLLAMA_URL} # es. http://localhost:11434 - model: nomic-embed-text-v2-moe - dim: 768 batch_size: 32 vectors: diff --git a/tools/tht/cmd/tht/main.go b/tools/tht/cmd/tht/main.go index 141220ad..73631971 100644 --- a/tools/tht/cmd/tht/main.go +++ b/tools/tht/cmd/tht/main.go @@ -528,8 +528,13 @@ func piCommand(ctx context.Context, installation config.Installation, runner com defer func() { _ = lock.Release() }() } if args[0] == "restart" || args[0] == "update" || args[0] == "rollback" { - if err := modelprojection.Generate(installation); err != nil { - return commandUsageError(stderr, "installation model catalog could not be projected: "+err.Error()) + drift, err := modelprojection.Check(installation) + if err != nil { + return commandUsageError(stderr, "installation model catalog could not be verified: "+err.Error()) + } + if len(drift) > 0 { + return commandUsageError(stderr, + "installation model catalog changed; run tht start to apply the complete runtime projection") } } controlled := compose.InstallationRunner{Installation: installation, Runner: runner} diff --git a/tools/tht/cmd/tht/main_test.go b/tools/tht/cmd/tht/main_test.go index 2b89fce3..4385a92b 100644 --- a/tools/tht/cmd/tht/main_test.go +++ b/tools/tht/cmd/tht/main_test.go @@ -1057,6 +1057,7 @@ func TestRunPiStatusUsesImageBundledPi(t *testing.T) { func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T) { fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture.setEnvironment(t) + fixture.generateModelProjection(t) if err := os.WriteFile(filepath.Join(fixture.projectDirectory, "docker", "core.Dockerfile"), []byte("ARG PI_VERSION=0.79.0\n"), 0o600); err != nil { t.Fatal(err) } @@ -1079,6 +1080,7 @@ func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) { fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture.setEnvironment(t) + fixture.generateModelProjection(t) var stdout, stderr bytes.Buffer exitCode := run(context.Background(), []string{"--installation", fixture.installationPath, "pi", "update", "--version", "0.81.0", "--source", "build"}, &stdout, &stderr) @@ -1095,6 +1097,7 @@ func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing func TestRunPiRestartRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) { fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture.setEnvironment(t) + fixture.generateModelProjection(t) var stdout, stderr bytes.Buffer exitCode := run(context.Background(), []string{ @@ -1117,6 +1120,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) { t.Fatal(err) } fixture.setEnvironment(t, secretPath) + fixture.generateModelProjection(t) t.Setenv("THT_FAKE_PI_VERSION", "pi-restart-secret") var stdout, stderr bytes.Buffer @@ -1138,6 +1142,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) { func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) { fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture.setEnvironment(t) + fixture.generateModelProjection(t) var stdout, stderr bytes.Buffer exitCode := run(context.Background(), []string{ @@ -1152,6 +1157,21 @@ func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) { } } +func TestRunPiRestartRejectsCatalogDriftBeforeInvokingDocker(t *testing.T) { + fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") + fixture.setEnvironment(t) + + var stdout, stderr bytes.Buffer + exitCode := run(context.Background(), []string{ + "--installation", fixture.installationPath, "pi", "restart", "--yes", + }, &stdout, &stderr) + + if exitCode != 2 || !strings.Contains(stderr.String(), "run tht start") { + t.Fatalf("run() = %d, stderr=%q; want complete projection guidance", exitCode, stderr.String()) + } + assertDockerNotInvoked(t, fixture) +} + func TestRunPiMaintenanceStatusAndRecoverConfirmationContract(t *testing.T) { fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture.setEnvironment(t) diff --git a/tools/tht/internal/config/installation_model_catalog_test.go b/tools/tht/internal/config/installation_model_catalog_test.go index ebdfa407..d2a0aa4a 100644 --- a/tools/tht/internal/config/installation_model_catalog_test.go +++ b/tools/tht/internal/config/installation_model_catalog_test.go @@ -78,6 +78,13 @@ func TestLoadRejectsInvalidModelCatalogDefaultsAndAuthentication(t *testing.T) { catalog: strings.Replace(validModelCatalogYAML(), "id: ollama/qwen3-embedding:0.6b", "id: external/qwen3-embedding:0.6b", 1), wantError: "ollama canonical id", }, + { + name: "upstream model aliases must be unique within a provider", + catalog: strings.Replace(validModelCatalogYAML(), + " models:\n glm-5.3:\n", + " models:\n glm-alias:\n upstreamModel: glm-5.3\n session:\n contextWindow: 200000\n maxTokens: 131072\n glm-5.3:\n", 1), + wantError: "same upstreamModel", + }, { name: "metadata default required", catalog: strings.Replace(validModelCatalogYAML(), " metadataGeneration: local/qwen\n", "", 1), diff --git a/tools/tht/internal/config/model_catalog.go b/tools/tht/internal/config/model_catalog.go index e8e9ecbb..f74201c5 100644 --- a/tools/tht/internal/config/model_catalog.go +++ b/tools/tht/internal/config/model_catalog.go @@ -14,6 +14,7 @@ const maxCatalogModels = 64 var catalogKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}$`) var catalogModelIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`) +var catalogUpstreamModelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`) var catalogAPIVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) // ModelCatalog is the only operator-authored source for model identity and runtime eligibility. @@ -163,11 +164,23 @@ func (c ModelCatalog) Validate(environment map[string]string) error { return err } hasSession, providerHasMetadata := false, false + upstreamModels := make(map[string]string) for modelID, model := range provider.Models { if !catalogKeyPattern.MatchString(modelID) { return fmt.Errorf("modelCatalog model %q/%q is invalid", providerID, modelID) } canonical := providerID + "/" + modelID + upstream := model.UpstreamModel + if upstream == "" { + upstream = modelID + } + if !catalogUpstreamModelPattern.MatchString(upstream) { + return fmt.Errorf("modelCatalog model %q upstreamModel is invalid", canonical) + } + if existing, duplicate := upstreamModels[upstream]; duplicate { + return fmt.Errorf("modelCatalog models %q and %q use the same upstreamModel", existing, canonical) + } + upstreamModels[upstream] = canonical if model.Session == nil && model.MetadataGeneration == nil { return fmt.Errorf("modelCatalog model %q has no runtime use", canonical) } diff --git a/tools/tht/internal/modelmigration/migration.go b/tools/tht/internal/modelmigration/migration.go index 93b8bf09..072d7f82 100644 --- a/tools/tht/internal/modelmigration/migration.go +++ b/tools/tht/internal/modelmigration/migration.go @@ -119,9 +119,10 @@ func Run(request Request) error { if !filepath.IsAbs(request.InstallationPath) || filepath.Base(request.InstallationPath) != "thothii-installation.yaml" { return errors.New("installation migration requires an absolute thothii-installation.yaml path") } - if !filepath.IsAbs(request.OutputPath) || filepath.Base(request.OutputPath) != "thothii-installation.yaml" || + outputExtension := strings.ToLower(filepath.Ext(request.OutputPath)) + if !filepath.IsAbs(request.OutputPath) || outputExtension != ".yaml" && outputExtension != ".yml" || filepath.Clean(request.OutputPath) == filepath.Clean(request.InstallationPath) { - return errors.New("installation migration output must be a different absolute thothii-installation.yaml path") + return errors.New("installation migration output must be a different absolute YAML path") } if request.SessionDefault == "" || request.EmbeddingID == "" || request.EmbeddingDimensions <= 0 { return errors.New("installation migration requires session default, embedding id, and positive embedding dimensions") diff --git a/tools/tht/internal/modelmigration/migration_test.go b/tools/tht/internal/modelmigration/migration_test.go index 7849f613..338d9e97 100644 --- a/tools/tht/internal/modelmigration/migration_test.go +++ b/tools/tht/internal/modelmigration/migration_test.go @@ -69,6 +69,19 @@ func TestRunReportsProviderAuthenticationConflictAndPublishesNothing(t *testing. } } +func TestRunAcceptsVersionedYAMLCandidateFilename(t *testing.T) { + fixture := migrationFixture(t, false) + request := fixture.request() + request.OutputPath = filepath.Join(filepath.Dir(request.OutputPath), "thothii-installation.v2.yaml") + + if err := Run(request); err != nil { + t.Fatalf("Run() error = %v", err) + } + if info, err := os.Stat(request.OutputPath); err != nil || !info.Mode().IsRegular() { + t.Fatalf("versioned candidate: info=%v err=%v", info, err) + } +} + type migrationTestFixture struct { legacy string output string diff --git a/tools/tht/internal/modelprojection/projection.go b/tools/tht/internal/modelprojection/projection.go index 9bd6874e..12225f72 100644 --- a/tools/tht/internal/modelprojection/projection.go +++ b/tools/tht/internal/modelprojection/projection.go @@ -21,6 +21,8 @@ const ( ComposeFile = "compose.models.yaml" ) +var renameProjectionDirectory = os.Rename + type runtimeCatalog struct { SchemaVersion int `json:"schemaVersion"` DefaultSession string `json:"defaultSession"` @@ -87,10 +89,11 @@ func Render(installation config.Installation) (map[string][]byte, error) { if model.Session == nil { continue } - enabled = append(enabled, model.ID) if model.SessionAdapter == nil || model.SessionAdapter.Mode != "openai_compatible" { + enabled = append(enabled, model.ID) continue } + enabled = append(enabled, model.Provider+"/"+model.UpstreamModel) provider := customProviders[model.Provider] provider.BaseURL = model.Endpoint.BaseURL provider.API = "openai-completions" @@ -137,52 +140,84 @@ func Render(installation config.Installation) (map[string][]byte, error) { }, nil } -// Generate atomically replaces each generated adapter after the complete candidate has rendered. +// Generate publishes all adapters as one directory generation. A failed replacement restores the +// previous directory, so callers never observe a successfully returned mixed generation. func Generate(installation config.Installation) error { artifacts, err := Render(installation) if err != nil { return err } - paths := sortedArtifactPaths(artifacts) - for _, relative := range paths { - destination := filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(relative)) - // The generated catalog contains references to secret environment variable names, never - // secret values. Core runs as an unprivileged container user and must be able to traverse - // the bind-mounted host directories and read the projections. + target := installation.GeneratedDirectory() + parent := filepath.Dir(target) + if err := os.MkdirAll(parent, 0o755); err != nil { + return fmt.Errorf("create model projection parent: %w", err) + } + candidate, err := os.MkdirTemp(parent, ".model-projections-candidate-*") + if err != nil { + return fmt.Errorf("create model projection candidate: %w", err) + } + defer func() { _ = os.RemoveAll(candidate) }() + if err := writeProjectionCandidate(candidate, artifacts); err != nil { + return err + } + + info, statErr := os.Lstat(target) + if os.IsNotExist(statErr) { + if err := renameProjectionDirectory(candidate, target); err != nil { + return fmt.Errorf("publish model projection generation: %w", err) + } + return nil + } + if statErr != nil { + return fmt.Errorf("inspect current model projection generation: %w", statErr) + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("current model projection path is not a regular directory") + } + previous, err := absentTemporaryPath(parent) + if err != nil { + return fmt.Errorf("reserve previous model projection generation: %w", err) + } + if err := renameProjectionDirectory(target, previous); err != nil { + return fmt.Errorf("prepare model projection generation replacement: %w", err) + } + if err := renameProjectionDirectory(candidate, target); err != nil { + if restoreErr := renameProjectionDirectory(previous, target); restoreErr != nil { + return fmt.Errorf("publish model projection generation: %v; restore previous generation: %w", err, restoreErr) + } + return fmt.Errorf("publish model projection generation: %w", err) + } + _ = os.RemoveAll(previous) + return nil +} + +func writeProjectionCandidate(directory string, artifacts map[string][]byte) error { + if err := os.Chmod(directory, 0o755); err != nil { + return fmt.Errorf("protect model projection candidate: %w", err) + } + for _, relative := range sortedArtifactPaths(artifacts) { + destination := filepath.Join(directory, filepath.FromSlash(relative)) if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { return fmt.Errorf("create model projection directory: %w", err) } - if err := os.Chmod(filepath.Dir(destination), 0o755); err != nil { - return fmt.Errorf("protect model projection directory: %w", err) - } - temporary, err := os.CreateTemp(filepath.Dir(destination), ".projection-*") - if err != nil { - return fmt.Errorf("create model projection candidate: %w", err) - } - temporaryName := temporary.Name() - published := false - defer func() { - if !published { - _ = os.Remove(temporaryName) - } - }() - if err := temporary.Chmod(0o644); err == nil { - _, err = temporary.Write(artifacts[relative]) - } - if closeErr := temporary.Close(); err == nil { - err = closeErr - } - if err != nil { + if err := os.WriteFile(destination, artifacts[relative], 0o644); err != nil { return fmt.Errorf("write model projection candidate: %w", err) } - if err := os.Rename(temporaryName, destination); err != nil { - return fmt.Errorf("publish model projection: %w", err) - } - published = true } return nil } +func absentTemporaryPath(parent string) (string, error) { + path, err := os.MkdirTemp(parent, ".model-projections-previous-*") + if err != nil { + return "", err + } + if err := os.Remove(path); err != nil { + return "", err + } + return path, nil +} + // Check returns relative artifact names whose current bytes differ from the catalog projection. func Check(installation config.Installation) ([]string, error) { artifacts, err := Render(installation) diff --git a/tools/tht/internal/modelprojection/projection_test.go b/tools/tht/internal/modelprojection/projection_test.go index 3aab0a32..1253d9ad 100644 --- a/tools/tht/internal/modelprojection/projection_test.go +++ b/tools/tht/internal/modelprojection/projection_test.go @@ -2,6 +2,7 @@ package modelprojection import ( "bytes" + "errors" "os" "path/filepath" "strings" @@ -41,9 +42,12 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T) t.Fatalf("Pi models projection must contain only custom providers:\n%s", piModels) } settings := string(first[PiSettingsFile]) - if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen"`) { + if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen2.5:7b"`) { t.Fatalf("Pi settings projection missing enabled session models:\n%s", settings) } + if !strings.Contains(piModels, `"id": "qwen2.5:7b"`) || !strings.Contains(catalog, `"id": "local/qwen"`) { + t.Fatalf("Pi projection must map canonical catalog keys to upstream model IDs") + } compose := string(first[ComposeFile]) for _, expected := range []string{"THT_MODEL_CATALOG_FILE", "THT_DEFAULT_SESSION_MODEL", "zai/glm-5.3", "THT_INTERNAL_EMBEDDING_ID", "THT_INTERNAL_EMBEDDING_MODEL", "OLLAMA_MODEL", installation.GeneratedModelCatalogPath()} { if !strings.Contains(compose, expected) { @@ -52,6 +56,46 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T) } } +func TestGenerateRestoresWholePreviousGenerationWhenPublishFails(t *testing.T) { + installation := projectionFixture(t) + if err := Generate(installation); err != nil { + t.Fatalf("Generate() initial error = %v", err) + } + before := make(map[string][]byte) + for _, path := range []string{CatalogFile, PiModelsFile, PiSettingsFile, ComposeFile} { + contents, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path))) + if err != nil { + t.Fatal(err) + } + before[path] = contents + } + provider := installation.ModelCatalog.Providers["zai"] + model := provider.Models["glm-5.3"] + model.Label = "Changed label" + provider.Models["glm-5.3"] = model + installation.ModelCatalog.Providers["zai"] = provider + + originalRename := renameProjectionDirectory + t.Cleanup(func() { renameProjectionDirectory = originalRename }) + renames := 0 + renameProjectionDirectory = func(oldPath, newPath string) error { + renames++ + if renames == 2 { + return errors.New("injected publish failure") + } + return os.Rename(oldPath, newPath) + } + if err := Generate(installation); err == nil { + t.Fatal("Generate() error = nil, want injected publish failure") + } + for path, expected := range before { + actual, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path))) + if err != nil || !bytes.Equal(actual, expected) { + t.Fatalf("artifact %q changed after failed generation: err=%v", path, err) + } + } +} + func TestGeneratePublishesAllArtifactsAndCheckDetectsDrift(t *testing.T) { installation := projectionFixture(t) if err := Generate(installation); err != nil { @@ -84,7 +128,7 @@ func projectionFixture(t *testing.T) config.Installation { Providers: map[string]config.ModelProvider{ "deepseek": {Authentication: config.ModelAuthentication{Mode: "pi_auth"}, Session: &config.ModelSessionAdapter{Mode: "pi_builtin"}, Models: map[string]config.CatalogModel{"deepseek-v4-pro": {Session: &config.SessionModel{}}}}, "zai": {Endpoint: &config.ModelEndpoint{BaseURL: "https://api.z.ai/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, Models: map[string]config.CatalogModel{"glm-5.3": {Label: "GLM-5.3", Session: &config.SessionModel{Reasoning: true, ContextWindow: 200000, MaxTokens: 131072}}}}, - "local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}}, + "local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {UpstreamModel: "qwen2.5:7b", Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}}, }, }, }