fix: harden model catalog projections

This commit is contained in:
Codex
2026-09-02 19:25:01 +02:00
parent ce4c31a6fb
commit a6a5bf2036
38 changed files with 573 additions and 83 deletions
+1
View File
@@ -268,6 +268,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
);
const listModels = deps?.listModels ?? createPiModelLister(config, {
modelCatalog: runtimeModelCatalog,
warn: (detail) => app.log.warn(
{ component: "pi-model-list", detail },
"Pi enabled-model configuration warning",
@@ -3,16 +3,17 @@ import type { CatalogDatabase } from "../repository.js";
const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$";
const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$";
const historicalOrCanonicalModelPattern = `(${legacyModelPattern})|(${canonicalModelPattern})`;
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
await sql.raw(`alter table description_generation_runs
drop constraint description_generation_runs_model_id_check,
add constraint description_generation_runs_model_id_check
check (model_id ~ '${canonicalModelPattern}')`).execute(db);
check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
await sql.raw(`alter table sensitive_data_suggestion_runs
drop constraint sensitive_data_suggestion_runs_model_id_check,
add constraint sensitive_data_suggestion_runs_model_id_check
check (model_id ~ '${canonicalModelPattern}')`).execute(db);
check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
}
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+17 -1
View File
@@ -10,6 +10,7 @@ import {
readConfiguredPiAgentFile,
validateDeclarativePiConfig,
} from "./managed-config.js";
import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
export interface PiModel {
provider: string;
@@ -30,6 +31,7 @@ interface Opts {
nowMs?: () => number;
loadEnabledModels?: () => PiEnabledModelsResult;
readModelsStore?: () => string | undefined;
modelCatalog?: RuntimeModelCatalog;
warn?: (message: string) => void;
}
@@ -82,9 +84,23 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModels
const byCompositeId = new Map(
available.map((model) => [`${model.provider}/${model.id}`, model]),
);
const catalogByPiId = new Map(
(opts.modelCatalog?.sessionModels() ?? []).map((model) => [
`${model.provider}/${model.upstreamModel}`,
model,
]),
);
const models = enabled.ids.flatMap((id) => {
const model = byCompositeId.get(id);
return model ? [model] : [];
if (!model) return [];
const catalogModel = catalogByPiId.get(id);
return [catalogModel ? {
...model,
provider: catalogModel.provider,
id: catalogModel.model,
name: catalogModel.label,
reasoning: catalogModel.session?.reasoning ?? model.reasoning,
} : model];
});
if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available");
cache = { at: now(), models };
+4 -1
View File
@@ -260,8 +260,11 @@ export class PiProcessManager {
const thinking = o.thinking ?? this.cfg.defaults.thinking;
if (provider && model) {
const upstreamModel = this.modelCatalog.sessionModels()
.find((entry) => entry.provider === provider && entry.model === model)
?.upstreamModel ?? model;
const response = await rt.rpc.request(
{ type: "set_model", provider, modelId: model } as object & { type: string },
{ type: "set_model", provider, modelId: upstreamModel } as object & { type: string },
);
rt.bridge.setContextWindow(response?.data?.contextWindow);
}
+2 -1
View File
@@ -79,6 +79,7 @@ export function createPiProviderSmoke(
|| catalog.defaultSession !== null;
const catalogModel = catalog.sessionModels()
.find((entry) => entry.provider === canonicalProvider && entry.model === model);
const upstreamModel = catalogModel?.upstreamModel ?? model;
const credentialName = catalogModel?.authentication.mode === "secret_env"
? catalogModel.authentication.apiKeyEnv
: undefined;
@@ -130,7 +131,7 @@ export function createPiProviderSmoke(
const capabilityGuard = failOnUnexpectedCapabilities(rpc);
const turn = async (): Promise<void> => {
requireSuccessfulResponse(await rpc.request({
type: "set_model", provider: canonicalProvider, modelId: model,
type: "set_model", provider: canonicalProvider, modelId: upstreamModel,
} as object & { type: string }));
requireSuccessfulResponse(await rpc.request({
type: "set_thinking_level", level: reasoning,
+1 -1
View File
@@ -140,7 +140,7 @@ function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbed
}
const model = requireString(embeddings, "model");
return {
id: `ollama/${model}`,
id: optionalString(embeddings, "id") ?? `ollama/${model}`,
model,
dimensions: requireNumber(embeddings, "dimensions"),
};
@@ -274,6 +274,8 @@ export function renderRuntimeConfig(
},
embedding: {
provider: "ollama_internal",
id: semanticRuntime.internalEmbeddingId
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
model: semanticRuntime.internalEmbeddingModel,
dimensions: semanticRuntime.internalEmbeddingDimensions,
},
@@ -287,6 +289,8 @@ export function renderRuntimeConfig(
embeddings: {
provider: "ollama_internal",
base_url: semanticRuntime.internalEmbeddingUrl,
id: semanticRuntime.internalEmbeddingId
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
model: semanticRuntime.internalEmbeddingModel,
dimensions: semanticRuntime.internalEmbeddingDimensions,
},
@@ -1,4 +1,5 @@
import { spawnSync } from "node:child_process";
import { randomUUID } from "node:crypto";
import { PostgreSqlContainer } from "@testcontainers/postgresql";
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
import { Pool } from "pg";
@@ -33,7 +34,30 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
const historicalDatabaseId = randomUUID();
await db.insertInto("workspaceDatabases").values({
id: historicalDatabaseId,
workspaceId: "migration-history",
engine: "postgres",
databaseName: "warehouse",
schemaName: "public",
}).execute();
await db.insertInto("descriptionGenerationRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
modelId: "openai-mini", language: "en", status: "completed", total: 1,
processed: 1, generated: 1,
}).execute();
await db.insertInto("sensitiveDataSuggestionRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
modelId: "openai-mini", status: "completed", total: 1,
suggestedSensitive: 1,
}).execute();
await upCanonicalModelIds(db);
await expect(db.insertInto("descriptionGenerationRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1,
processed: 1, generated: 1,
}).execute()).resolves.toBeDefined();
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
await upRuntimeSequencePrivileges(db);
const sequencePrivilege = await sql<{ allowed: boolean }>`
+31
View File
@@ -6,6 +6,7 @@ import { join } from "node:path";
import path from "node:path";
import { createPiModelLister } from "../src/pi/list-models.js";
import { loadConfig } from "../src/config.js";
import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
@@ -44,6 +45,36 @@ test("createPiModelLister returns mapped PiModel[] from get_available_models", a
}
});
test("catalog listing translates upstream Pi IDs back to canonical model keys", async () => {
const script = scriptWith([
{ provider: "local", id: "qwen2.5:7b", name: "Upstream label", reasoning: false },
]);
const model: RuntimeModel = {
id: "local/qwen", provider: "local", model: "qwen", label: "Catalog Qwen",
upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 },
};
const modelCatalog: RuntimeModelCatalog = {
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
};
try {
const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
...noManagedModels,
modelCatalog,
loadEnabledModels: enabled("local/qwen2.5:7b"),
spawnFn: () => spawn("node", [FAKE, script]) as any,
});
await expect(lister()).resolves.toEqual([{
provider: "local", id: "qwen", name: "Catalog Qwen", reasoning: true,
}]);
} finally {
rmSync(path.dirname(script), { recursive: true, force: true });
}
});
test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
try {
+48
View File
@@ -718,6 +718,54 @@ test.each([["OpenAI", "openai"], ["gemini", "google"]])(
},
);
test("set_model translates a canonical catalog key to its upstream Pi model ID", async () => {
const root = mkdtempSync(path.join(tmpdir(), "thothii-upstream-model-"));
const agentDir = path.join(root, "agent");
mkdirSync(agentDir, { mode: 0o700 });
writeFileSync(path.join(agentDir, "models.json"), JSON.stringify({
providers: {
local: {
baseUrl: "http://ollama:11434/v1", apiKey: "local",
models: [{ id: "qwen2.5:7b" }],
},
},
}), { mode: 0o600 });
vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
const child = recordingChild();
child.stderr.resume = () => {};
child.stdin.write = (data: unknown) => {
const request = JSON.parse(String(data));
child._writes.push(String(data));
if (request.id) {
queueMicrotask(() => child.stdout.emit("data", `${JSON.stringify({
type: "response", id: request.id, success: true,
})}\n`));
}
return true;
};
const model: RuntimeModel = {
id: "local/qwen", provider: "local", model: "qwen", label: "Qwen",
upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
session: { reasoning: false, contextWindow: 32768, maxTokens: 8192 },
};
const modelCatalog: RuntimeModelCatalog = {
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
};
const mgr = new PiProcessManager(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
modelCatalog, authProviders: () => new Set(), spawnFn: () => child as any,
});
try {
await mgr.spawnFor("upstream-model", { provider: "local", model: "qwen" });
expect(child._writes.join("")).toContain('"modelId":"qwen2.5:7b"');
} finally {
mgr.teardown("upstream-model");
vi.unstubAllEnvs();
rmSync(root, { recursive: true, force: true });
}
});
test.each(["installation-local", "private-compatible"])(
"provider %s configured with a literal apiKey spawns without a managed key",
async (provider) => {
+12 -1
View File
@@ -300,11 +300,22 @@ test("provider smoke makes one configured request from an isolated no-capability
});
}
});
const smokeModel: RuntimeModel = {
id: "zai/catalog-glm", provider: "zai", model: "catalog-glm", label: "GLM",
upstreamModel: "glm-5.2", authentication: { mode: "pi_auth" },
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true },
};
const smokeCatalog: RuntimeModelCatalog = {
defaultSession: smokeModel.id, defaultMetadataGeneration: null, embedding: null,
sessionModels: () => [smokeModel], metadataModels: () => [],
hasSession: (id) => id === smokeModel.id,
};
const smoke = createPiProviderSmoke(loadConfig({
THT_HARNESS_DIR: "/app/harness",
PI_BIN: "/usr/local/bin/pi",
THT_DATA_ROOT: "/mounted-workflow-state",
}), {
modelCatalog: smokeCatalog,
spawnFn: (...args) => {
spawns.push(args);
expect(args[2].cwd).not.toBe("/app/harness");
@@ -324,7 +335,7 @@ test("provider smoke makes one configured request from an isolated no-capability
});
await expect(smoke({
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750,
provider: "zai", model: "catalog-glm", reasoning: "medium", timeoutMs: 750,
})).resolves.toBeUndefined();
expect(spawns).toHaveLength(1);
expect(spawns[0][0]).toBe("/usr/local/bin/pi");
@@ -73,6 +73,7 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
embeddings: {
provider: "ollama_internal", base_url: "http://embedding:11434",
id: "ollama/qwen3-embedding:0.6b",
model: "qwen3-embedding:0.6b", dimensions: 1024,
},
},
@@ -35,7 +35,6 @@ resources:
base_url: http://qdrant:6333
collection: server-sessions
embeddings:
# id, model, and dimensions are injected from the Installation Model Catalog.
provider: ollama_internal
base_url: http://embedding:11434
model: qwen3-embedding:0.6b
dimensions: 1024
+9 -7
View File
@@ -100,7 +100,7 @@ generated/
├── pi/
│ ├── models.json
│ └── settings.json
└── compose.model-catalog.yaml
└── compose.models.yaml
```
The normalized catalog is consumed by the backend. The Pi files and Compose override are boundary
@@ -117,17 +117,19 @@ tht --installation "$INSTALLATION" start
tht --installation "$INSTALLATION" doctor
```
After editing `modelCatalog` or provider credentials, reload the current Pi image. Restart validates
the YAML and regenerates projections before recreating `core`:
After editing `modelCatalog` or provider credentials, apply the complete runtime projection with
the normal installation lifecycle, then run the Pi checks:
```bash
tht --installation "$INSTALLATION" pi restart --yes --drain
tht --installation "$INSTALLATION" start
tht --installation "$INSTALLATION" pi doctor
tht --installation "$INSTALLATION" pi test
```
`tht pi update` changes the Pi version; it is not the configuration command. There is no
`tht pi configure` and no separate apply command.
`tht pi restart`, `tht pi update`, and `tht pi rollback` refuse to run while generated model
projections differ from `modelCatalog`: those commands recreate only `core`, so they must never
partially apply an embedding change. `tht pi update` changes the Pi version; it is not the
configuration command. There is no `tht pi configure` and no separate apply command.
## Migrating a legacy installation
@@ -136,7 +138,7 @@ files, but never modifies them. Supply the facts that cannot be inferred safely
candidate:
```bash
tht --installation /absolute/path/legacy-installation.yaml installation migrate \
tht --installation /absolute/path/legacy/thothii-installation.yaml installation migrate \
--output /absolute/path/thothii-installation.v2.yaml \
--session-default zai/glm-5.3 \
--embedding-id ollama/qwen3-embedding:0.6b \
@@ -166,6 +166,8 @@ Generation is deterministic and published only after every candidate artifact va
generation aborts start before Compose is invoked. `tht doctor` recomputes expected bytes and reports
differences; no digest manifest or separate apply command exists. When projection bytes change,
`tht start` recreates the affected services so they cannot continue with an older bind mount.
Pi-only restart, update, and rollback operations reject projection drift and direct the operator to
`tht start`, because applying only the core-facing files could leave embedding services stale.
Generated projections are not backed up. Restore validates the canonical installation descriptor,
regenerates every projection, and only then starts services. Base Compose files and `operator.env`
+57 -5
View File
@@ -214,7 +214,7 @@ embeddings: {provider: ollama_internal, base_url: http://embedding:11434, model:
assert cfg.vectors.writer.api_key == "writer"
def test_accepts_only_internal_ollama_embedding_contract(tmp_path):
def test_accepts_catalog_selected_internal_ollama_embedding_contract(tmp_path):
workspace = tmp_path / "workspace.yaml"
workspace.write_text(
"""
@@ -225,8 +225,9 @@ resources:
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
model: qwen3-embedding:0.6b
dimensions: 1024
id: ollama/bge-m3
model: bge-m3
dimensions: 1536
"""
)
@@ -234,8 +235,59 @@ resources:
assert cfg.embeddings.provider == "ollama_internal"
assert cfg.embeddings.base_url == "http://embedding:11434"
assert cfg.embeddings.model == "qwen3-embedding:0.6b"
assert cfg.embeddings.dim == 1024
assert cfg.embeddings.id == "ollama/bge-m3"
assert cfg.embeddings.model == "bge-m3"
assert cfg.embeddings.dim == 1536
def test_installation_embedding_projection_completes_model_free_runtime_source(
tmp_path, monkeypatch,
):
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536")
workspace = tmp_path / "workspace.yaml"
workspace.write_text(
"""
dwh:
type: postgres_direct
connection: {database: analytics, schema: mart, user: reader, password: secret}
resources:
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
"""
)
cfg = load_config(workspace)
assert cfg.embeddings.id == "ollama/bge-m3"
assert cfg.embeddings.model == "bge-m3"
assert cfg.embeddings.dim == 1536
def test_workspace_embedding_values_cannot_override_installation_projection(
tmp_path, monkeypatch,
):
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536")
workspace = tmp_path / "workspace.yaml"
workspace.write_text(
"""
dwh:
type: postgres_direct
connection: {database: analytics, schema: mart, user: reader, password: secret}
resources:
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
model: workspace-owned-model
"""
)
with pytest.raises(ConfigError, match="proprietà dell'installazione|diverge"):
load_config(workspace)
def test_accepts_internal_qdrant_resource_contract(tmp_path):
+21
View File
@@ -40,6 +40,7 @@ def test_manifest_contains_provenance_without_credentials():
manifest_id="manifest:abc",
created_at=datetime(2026, 7, 12, tzinfo=UTC),
pipeline_version="evidence-v1",
embedding_id="ollama/nomic-embed-text",
embedding_model="nomic-embed-text",
embedding_dimensions=768,
documents=[document()],
@@ -52,6 +53,7 @@ def test_manifest_contains_provenance_without_credentials():
assert "etag:abc" in payload
assert "evidence-v1" in payload
assert "nomic-embed-text" in payload
assert '"embedding_id":"ollama/nomic-embed-text"' in payload
assert "api_key" not in payload
@@ -93,6 +95,25 @@ def test_manifest_validates_embedding_compatibility_fields():
)
def test_schema_v1_manifest_without_canonical_embedding_id_remains_readable():
manifest = CorpusManifest.model_validate({
"schema_version": 1,
"embedding_model": "legacy-model",
"embedding_dimensions": 768,
})
assert manifest.embedding_id is None
def test_schema_v2_embedding_generation_requires_canonical_id():
with pytest.raises(ValidationError, match="embedding_id"):
CorpusManifest(
schema_version=2,
embedding_model="model-v2",
embedding_dimensions=768,
)
def test_canonical_metadata_rejects_secrets_and_non_json_values():
with pytest.raises(ValidationError, match="credential-like"):
CanonicalDocument.model_validate(
+21 -1
View File
@@ -105,11 +105,13 @@ def item(name, fingerprint):
)
def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None,
def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a",
embedding_id=None, policy=None,
retain=3, candidate_evaluator=None):
return CorpusPipeline(
store=CorpusStore(tmp_path / "corpus"), sources=[source],
embedder=embedder or Embedder(), vector_store=vectors or Vectors(),
embedding_id=embedding_id,
embedding_model=model, embedding_dimensions=3,
chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100),
pipeline_version="evidence-v1",
@@ -875,6 +877,24 @@ def test_model_or_chunk_policy_change_forces_full_rebuild(tmp_path):
assert source.acquire_calls == ["fs:one"]
def test_canonical_embedding_id_change_forces_full_rebuild_and_is_persisted(tmp_path):
one = item("one", "a")
first = pipeline(
tmp_path, Source([(one, "hello")]), model="same-upstream",
embedding_id="ollama/catalog-a",
).run()
assert first.manifest.embedding_id == "ollama/catalog-a"
source = Source([(one, "hello")])
changed = pipeline(
tmp_path, source, model="same-upstream", embedding_id="ollama/catalog-b",
).run()
assert changed.changed == ("fs:one",)
assert changed.manifest.embedding_id == "ollama/catalog-b"
assert source.acquire_calls == ["fs:one"]
def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path):
one = item("one", "a")
good = pipeline(tmp_path, Source([(one, "old")]))
+16 -2
View File
@@ -21,7 +21,8 @@ def _write_cfg(tmp_path, raw):
return cfg
def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid", collection="psd", model="qwen3-embedding:0.6b"):
def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid",
collection="psd", model="qwen3-embedding:0.6b", dimensions=1024):
return {
"schemaVersion": 1,
"workspace": {"schema_version": 3, "id": "psd", "name": "PSD", "language": "it"},
@@ -34,7 +35,10 @@ def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.inval
"connection": {"host": "h", "port": 5432, "database": "warehouse", "schema": "dw", "user": "reader", "password": "secret"},
},
"vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": collection, "collection_lifecycle": "self_heal"},
"embeddings": {"provider": "ollama_internal", "base_url": "http://embedding:11434", "model": model, "dimensions": 1024},
"embeddings": {
"provider": "ollama_internal", "base_url": "http://embedding:11434",
"id": f"ollama/{model}", "model": model, "dimensions": dimensions,
},
"roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")},
"paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")},
}
@@ -52,6 +56,16 @@ def test_canonical_json_is_deterministic_and_key_ordered(cfg):
assert keys == ["schemaVersion", "dwh", "vector", "embedding", "roots"]
def test_catalog_embedding_identity_and_dimensions_drive_effective_config(tmp_path):
cfg = _write_cfg(tmp_path, _cfg(tmp_path, model="bge-m3", dimensions=1536))
document = __import__("json").loads(canonical_effective_config_json(cfg))
assert document["embedding"] == {
"id": "ollama/bge-m3", "model": "bge-m3", "dimensions": 1536,
}
assert document["vector"]["dimensions"] == 1536
def test_canonical_excludes_credentials_and_evidence(cfg):
doc = canonical_effective_config_json(cfg)
assert "secret" not in doc
@@ -149,6 +149,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk
"sources": [object()],
"embedder": object(),
"vector_store": object(),
"embedding_id": "ollama/model",
"embedding_model": "model",
"embedding_dimensions": 3,
"chunk_policy": object(),
+2
View File
@@ -10,6 +10,8 @@ from tht.config import EmbeddingsConfig
def _cfg(**kw):
kw.setdefault("model", "nomic-embed-text-v2-moe")
kw.setdefault("dim", 768)
emb = EmbeddingsConfig(base_url="http://localhost:11434", **kw)
return SimpleNamespace(embeddings=emb)
+3
View File
@@ -284,6 +284,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
sources,
embedder,
vector_store,
embedding_id,
embedding_model,
embedding_dimensions,
chunk_policy,
@@ -293,6 +294,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
candidate_evaluator,
):
calls["init"] = {
"embedding_id": embedding_id,
"embedding_model": embedding_model,
"embedding_dimensions": embedding_dimensions,
"pipeline_version": pipeline_version,
@@ -314,6 +316,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
command.run_from_config(config)
assert calls["init"]["embedding_id"] == "ollama/qwen3-embedding:0.6b"
assert calls["init"]["sparse_language"] == "english"
assert calls["init"]["candidate_evaluator"] is None
assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
+2
View File
@@ -187,6 +187,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
embedder=embedder,
vector_store=vector_store,
embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}",
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1",
@@ -222,6 +223,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
pipeline = build_preprocessing_pipeline(
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True),
embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}",
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1",
+84 -13
View File
@@ -60,8 +60,12 @@ def canonical_effective_config_document(cfg) -> dict:
return {
"schemaVersion": 1,
"dwh": dwh,
"vector": {"collection": collection, "dimensions": 1024, "distance": "cosine"},
"embedding": {"model": model, "dimensions": int(embed_dim)},
"vector": {"collection": collection, "dimensions": int(embed_dim), "distance": "cosine"},
"embedding": {
"id": getattr(embeddings, "id", None) or f"ollama/{model}",
"model": model,
"dimensions": int(embed_dim),
},
"roots": {
"artifacts": str(getattr(cfg.paths, "artifacts", Path("artifacts"))),
"indexes": str(getattr(cfg.paths, "indexes", Path("indexes"))),
@@ -499,8 +503,9 @@ class EvidenceSourcesConfig(BaseModel):
class EmbeddingsConfig(BaseModel):
provider: str = "ollama_internal"
base_url: str
model: str = "nomic-embed-text-v2-moe"
dim: int = Field(default=768, alias="dimensions")
id: str | None = None
model: str
dim: int = Field(alias="dimensions")
batch_size: int = 16
timeout: int = 300
connect_timeout: int = 5
@@ -642,6 +647,7 @@ def load_config(path: Path) -> Config:
if not isinstance(raw, dict):
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
expanded = _resolve_secret_files(_resolve_evidence_secret_files(_expand_env(raw)))
_apply_installation_embedding_projection(expanded, path)
_validate_internal_embedding_contract(expanded, path)
_validate_internal_vector_contract(expanded, path)
translated, used_legacy = translate_legacy_config(expanded)
@@ -710,6 +716,56 @@ def load_config(path: Path) -> Config:
return cfg
def _apply_installation_embedding_projection(raw: dict[str, Any], path: Path) -> None:
projected = {
"id": os.environ.get("THT_INTERNAL_EMBEDDING_ID"),
"model": os.environ.get("THT_INTERNAL_EMBEDDING_MODEL"),
"dimensions": os.environ.get("THT_INTERNAL_EMBEDDING_DIMENSIONS"),
}
if all(value is None for value in projected.values()):
return
if any(value is None for value in projected.values()):
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"la proiezione embedding dell'installazione è incompleta"
)
embedding_id = projected["id"]
model = projected["model"]
try:
dimensions = int(projected["dimensions"] or "")
except ValueError:
dimensions = 0
if embedding_id != f"ollama/{model}" or dimensions <= 0:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"la proiezione embedding dell'installazione non è canonica"
)
sections: list[tuple[dict[str, Any], str]] = []
embeddings = raw.get("embeddings")
if isinstance(embeddings, dict):
sections.append((embeddings, "dim"))
resources = raw.get("resources")
if isinstance(resources, dict) and isinstance(resources.get("embeddings"), dict):
sections.append((resources["embeddings"], "dimensions"))
for section, dimension_key in sections:
declared = {
"id": section.get("id"),
"model": section.get("model"),
"dimensions": section.get(dimension_key),
}
expected = {"id": embedding_id, "model": model, "dimensions": dimensions}
for key, value in declared.items():
if value is not None and str(value) != str(expected[key]):
raise ConfigError(
f"Configurazione non valida in {path}:\n"
f"{key} è proprietà dell'installazione e diverge dalla proiezione attiva"
)
section["id"] = embedding_id
section["model"] = model
section[dimension_key] = dimensions
def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> None:
resources = raw.get("resources")
if not isinstance(resources, dict):
@@ -720,9 +776,10 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
provider = embeddings.get("provider")
model = embeddings.get("model")
embedding_id = embeddings.get("id")
dimensions = embeddings.get("dimensions")
base_url = embeddings.get("base_url")
allowed = {"provider", "base_url", "model", "dimensions"}
allowed = {"provider", "base_url", "id", "model", "dimensions"}
unexpected = sorted(set(embeddings) - allowed)
if unexpected:
raise ConfigError(
@@ -734,15 +791,24 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
f"Configurazione non valida in {path}:\n"
"resources.embeddings.provider deve essere 'ollama_internal'"
)
if model != "qwen3-embedding:0.6b":
if not isinstance(model, str) or not model:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"resources.embeddings.model deve essere 'qwen3-embedding:0.6b'"
"resources.embeddings.model deve essere valorizzato"
)
if dimensions != 1024:
try:
parsed_dimensions = int(dimensions)
except (TypeError, ValueError):
parsed_dimensions = 0
if isinstance(dimensions, bool) or parsed_dimensions <= 0:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"resources.embeddings.dimensions deve essere 1024"
"resources.embeddings.dimensions deve essere un intero positivo"
)
if embedding_id is not None and embedding_id != f"ollama/{model}":
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"resources.embeddings.id deve essere l'identità canonica ollama/<model>"
)
if not _is_allowed_internal_embedding_url(base_url):
raise ConfigError(
@@ -805,15 +871,20 @@ def _validate_active_embeddings_config(
f"Configurazione non valida in {path}:\n"
"embeddings.provider deve essere 'ollama_internal'"
)
if embeddings.model != "qwen3-embedding:0.6b":
if not embeddings.model:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"embeddings.model deve essere 'qwen3-embedding:0.6b'"
"embeddings.model deve essere valorizzato"
)
if embeddings.dim != 1024:
if embeddings.dim <= 0:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"embeddings.dim deve essere 1024"
"embeddings.dim deve essere un intero positivo"
)
if embeddings.id is not None and embeddings.id != f"ollama/{embeddings.model}":
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"embeddings.id deve essere l'identità canonica ollama/<model>"
)
if not _is_allowed_internal_embedding_url(embeddings.base_url):
raise ConfigError(
+17
View File
@@ -19,6 +19,9 @@ from tht.evidence.contracts import (
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
_EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$")
_CANONICAL_MODEL_ID = re.compile(
r"^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$"
)
_EVIDENCE_METADATA_KEYS = frozenset({
"evidence_id", "evidence_kind", "purposes", "scope", "language", "provenance",
})
@@ -37,6 +40,12 @@ def _validate_hash(value: str) -> str:
return value
def _validate_canonical_model_id(value: str) -> str:
if not _CANONICAL_MODEL_ID.fullmatch(value):
raise ValueError("model identifier must use canonical provider/model form")
return value
def _require_content_hash(content: str, content_hash: str) -> None:
expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
if content_hash != expected:
@@ -146,6 +155,7 @@ class CorpusManifest(_WithMetadata):
manifest_id: str | None = None
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
pipeline_version: str = Field(default="evidence-v1", min_length=1)
embedding_id: str | None = None
embedding_model: str | None = None
embedding_dimensions: int | None = Field(default=None, gt=0)
vector_generation: str | None = None
@@ -155,6 +165,9 @@ class CorpusManifest(_WithMetadata):
_manifest_id = field_validator("manifest_id")(
lambda value: _validate_namespaced_id(value) if value is not None else None
)
_embedding_id = field_validator("embedding_id")(
lambda value: _validate_canonical_model_id(value) if value is not None else None
)
_vector_generation = field_validator("vector_generation")(
lambda value: _validate_namespaced_id(value) if value is not None else None
)
@@ -164,6 +177,10 @@ class CorpusManifest(_WithMetadata):
def validate_generation(self) -> "CorpusManifest":
if (self.embedding_model is None) != (self.embedding_dimensions is None):
raise ValueError("embedding_model and embedding_dimensions must be set together")
if self.embedding_id is not None and self.embedding_model is None:
raise ValueError("embedding_id requires embedding model and dimensions")
if self.schema_version >= 2 and self.embedding_model is not None and self.embedding_id is None:
raise ValueError("schema version 2 embedding generations require embedding_id")
if self.vector_generation is not None and self.embedding_model is None:
raise ValueError("vector_generation requires embedding model and dimension compatibility")
+10 -1
View File
@@ -125,6 +125,7 @@ class CorpusPipeline:
def __init__(
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
embedding_id: str | None = None,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str | None = None, sparse_language: str = "italian",
candidate_evaluator: Callable[[CorpusManifest], object] | None = None,
@@ -133,6 +134,7 @@ class CorpusPipeline:
self.sources = sources
self.embedder = embedder
self.vector_store = vector_store
self.embedding_id = embedding_id or f"ollama/{embedding_model}"
self.embedding_model = embedding_model
self.embedding_dimensions = embedding_dimensions
self.chunk_policy = chunk_policy
@@ -284,6 +286,7 @@ class CorpusPipeline:
source_by_id = {item.source_id: (source, item) for source, item in discovered}
compatibility = _fingerprint({
"pipeline": self.pipeline_version,
"embedding_id": self.embedding_id,
"model": self.embedding_model,
"dimensions": self.embedding_dimensions,
"chunk_policy": asdict(self.chunk_policy),
@@ -294,6 +297,7 @@ class CorpusPipeline:
"compatibility_fingerprint": compatibility,
"pipeline_version": self.pipeline_version,
"chunk_policy_version": self.chunk_policy.version,
"embedding_id": self.embedding_id,
"embedding_model": self.embedding_model,
"embedding_dimensions": self.embedding_dimensions,
}
@@ -492,7 +496,9 @@ class CorpusPipeline:
) for document in documents
}
manifest = CorpusManifest(
schema_version=2,
pipeline_version=self.pipeline_version,
embedding_id=self.embedding_id,
embedding_model=self.embedding_model,
embedding_dimensions=self.embedding_dimensions,
vector_generation=plan["generation"],
@@ -723,7 +729,8 @@ class CorpusPipeline:
prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {}
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
compatibility = _fingerprint({
"pipeline": self.pipeline_version, "model": self.embedding_model,
"pipeline": self.pipeline_version, "embedding_id": self.embedding_id,
"model": self.embedding_model,
"dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy),
})
previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None
@@ -761,7 +768,9 @@ class CorpusPipeline:
for document in documents
}
manifest = CorpusManifest(
schema_version=2,
pipeline_version=self.pipeline_version,
embedding_id=self.embedding_id,
embedding_model=self.embedding_model,
embedding_dimensions=self.embedding_dimensions,
vector_generation=generation,
+2
View File
@@ -22,6 +22,7 @@ def build_preprocessing_pipeline(
vector_store: VectorStore,
embedding_model: str,
embedding_dimensions: int,
embedding_id: str | None = None,
chunk_policy: ChunkPolicy,
pipeline_version: str,
retain_published_generations: int = 3,
@@ -35,6 +36,7 @@ def build_preprocessing_pipeline(
sources=sources,
embedder=embedder,
vector_store=vector_store,
embedding_id=embedding_id,
embedding_model=embedding_model,
embedding_dimensions=embedding_dimensions,
chunk_policy=chunk_policy,
-2
View File
@@ -39,8 +39,6 @@ evidence:
embeddings:
base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434
model: nomic-embed-text-v2-moe
dim: 768
batch_size: 32
# Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che
-2
View File
@@ -25,8 +25,6 @@ paths:
embeddings:
base_url: ${THT_OLLAMA_URL}
model: nomic-embed-text-v2-moe
dim: 768
batch_size: 64
# LOADING diretto del pgvector (server-only). Su workstation la lettura passa da
-2
View File
@@ -39,8 +39,6 @@ evidence:
embeddings:
base_url: ${THT_OLLAMA_URL} # es. http://localhost:11434
model: nomic-embed-text-v2-moe
dim: 768
batch_size: 32
vectors:
+7 -2
View File
@@ -528,8 +528,13 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
defer func() { _ = lock.Release() }()
}
if args[0] == "restart" || args[0] == "update" || args[0] == "rollback" {
if err := modelprojection.Generate(installation); err != nil {
return commandUsageError(stderr, "installation model catalog could not be projected: "+err.Error())
drift, err := modelprojection.Check(installation)
if err != nil {
return commandUsageError(stderr, "installation model catalog could not be verified: "+err.Error())
}
if len(drift) > 0 {
return commandUsageError(stderr,
"installation model catalog changed; run tht start to apply the complete runtime projection")
}
}
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
+20
View File
@@ -1057,6 +1057,7 @@ func TestRunPiStatusUsesImageBundledPi(t *testing.T) {
func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t)
fixture.generateModelProjection(t)
if err := os.WriteFile(filepath.Join(fixture.projectDirectory, "docker", "core.Dockerfile"), []byte("ARG PI_VERSION=0.79.0\n"), 0o600); err != nil {
t.Fatal(err)
}
@@ -1079,6 +1080,7 @@ func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T
func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t)
fixture.generateModelProjection(t)
var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{"--installation", fixture.installationPath, "pi", "update", "--version", "0.81.0", "--source", "build"}, &stdout, &stderr)
@@ -1095,6 +1097,7 @@ func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing
func TestRunPiRestartRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t)
fixture.generateModelProjection(t)
var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{
@@ -1117,6 +1120,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) {
t.Fatal(err)
}
fixture.setEnvironment(t, secretPath)
fixture.generateModelProjection(t)
t.Setenv("THT_FAKE_PI_VERSION", "pi-restart-secret")
var stdout, stderr bytes.Buffer
@@ -1138,6 +1142,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) {
func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t)
fixture.generateModelProjection(t)
var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{
@@ -1152,6 +1157,21 @@ func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) {
}
}
func TestRunPiRestartRejectsCatalogDriftBeforeInvokingDocker(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t)
var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{
"--installation", fixture.installationPath, "pi", "restart", "--yes",
}, &stdout, &stderr)
if exitCode != 2 || !strings.Contains(stderr.String(), "run tht start") {
t.Fatalf("run() = %d, stderr=%q; want complete projection guidance", exitCode, stderr.String())
}
assertDockerNotInvoked(t, fixture)
}
func TestRunPiMaintenanceStatusAndRecoverConfirmationContract(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t)
@@ -78,6 +78,13 @@ func TestLoadRejectsInvalidModelCatalogDefaultsAndAuthentication(t *testing.T) {
catalog: strings.Replace(validModelCatalogYAML(), "id: ollama/qwen3-embedding:0.6b", "id: external/qwen3-embedding:0.6b", 1),
wantError: "ollama canonical id",
},
{
name: "upstream model aliases must be unique within a provider",
catalog: strings.Replace(validModelCatalogYAML(),
" models:\n glm-5.3:\n",
" models:\n glm-alias:\n upstreamModel: glm-5.3\n session:\n contextWindow: 200000\n maxTokens: 131072\n glm-5.3:\n", 1),
wantError: "same upstreamModel",
},
{
name: "metadata default required",
catalog: strings.Replace(validModelCatalogYAML(), " metadataGeneration: local/qwen\n", "", 1),
@@ -14,6 +14,7 @@ const maxCatalogModels = 64
var catalogKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}$`)
var catalogModelIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`)
var catalogUpstreamModelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`)
var catalogAPIVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
// ModelCatalog is the only operator-authored source for model identity and runtime eligibility.
@@ -163,11 +164,23 @@ func (c ModelCatalog) Validate(environment map[string]string) error {
return err
}
hasSession, providerHasMetadata := false, false
upstreamModels := make(map[string]string)
for modelID, model := range provider.Models {
if !catalogKeyPattern.MatchString(modelID) {
return fmt.Errorf("modelCatalog model %q/%q is invalid", providerID, modelID)
}
canonical := providerID + "/" + modelID
upstream := model.UpstreamModel
if upstream == "" {
upstream = modelID
}
if !catalogUpstreamModelPattern.MatchString(upstream) {
return fmt.Errorf("modelCatalog model %q upstreamModel is invalid", canonical)
}
if existing, duplicate := upstreamModels[upstream]; duplicate {
return fmt.Errorf("modelCatalog models %q and %q use the same upstreamModel", existing, canonical)
}
upstreamModels[upstream] = canonical
if model.Session == nil && model.MetadataGeneration == nil {
return fmt.Errorf("modelCatalog model %q has no runtime use", canonical)
}
@@ -119,9 +119,10 @@ func Run(request Request) error {
if !filepath.IsAbs(request.InstallationPath) || filepath.Base(request.InstallationPath) != "thothii-installation.yaml" {
return errors.New("installation migration requires an absolute thothii-installation.yaml path")
}
if !filepath.IsAbs(request.OutputPath) || filepath.Base(request.OutputPath) != "thothii-installation.yaml" ||
outputExtension := strings.ToLower(filepath.Ext(request.OutputPath))
if !filepath.IsAbs(request.OutputPath) || outputExtension != ".yaml" && outputExtension != ".yml" ||
filepath.Clean(request.OutputPath) == filepath.Clean(request.InstallationPath) {
return errors.New("installation migration output must be a different absolute thothii-installation.yaml path")
return errors.New("installation migration output must be a different absolute YAML path")
}
if request.SessionDefault == "" || request.EmbeddingID == "" || request.EmbeddingDimensions <= 0 {
return errors.New("installation migration requires session default, embedding id, and positive embedding dimensions")
@@ -69,6 +69,19 @@ func TestRunReportsProviderAuthenticationConflictAndPublishesNothing(t *testing.
}
}
func TestRunAcceptsVersionedYAMLCandidateFilename(t *testing.T) {
fixture := migrationFixture(t, false)
request := fixture.request()
request.OutputPath = filepath.Join(filepath.Dir(request.OutputPath), "thothii-installation.v2.yaml")
if err := Run(request); err != nil {
t.Fatalf("Run() error = %v", err)
}
if info, err := os.Stat(request.OutputPath); err != nil || !info.Mode().IsRegular() {
t.Fatalf("versioned candidate: info=%v err=%v", info, err)
}
}
type migrationTestFixture struct {
legacy string
output string
@@ -21,6 +21,8 @@ const (
ComposeFile = "compose.models.yaml"
)
var renameProjectionDirectory = os.Rename
type runtimeCatalog struct {
SchemaVersion int `json:"schemaVersion"`
DefaultSession string `json:"defaultSession"`
@@ -87,10 +89,11 @@ func Render(installation config.Installation) (map[string][]byte, error) {
if model.Session == nil {
continue
}
enabled = append(enabled, model.ID)
if model.SessionAdapter == nil || model.SessionAdapter.Mode != "openai_compatible" {
enabled = append(enabled, model.ID)
continue
}
enabled = append(enabled, model.Provider+"/"+model.UpstreamModel)
provider := customProviders[model.Provider]
provider.BaseURL = model.Endpoint.BaseURL
provider.API = "openai-completions"
@@ -137,52 +140,84 @@ func Render(installation config.Installation) (map[string][]byte, error) {
}, nil
}
// Generate atomically replaces each generated adapter after the complete candidate has rendered.
// Generate publishes all adapters as one directory generation. A failed replacement restores the
// previous directory, so callers never observe a successfully returned mixed generation.
func Generate(installation config.Installation) error {
artifacts, err := Render(installation)
if err != nil {
return err
}
paths := sortedArtifactPaths(artifacts)
for _, relative := range paths {
destination := filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(relative))
// The generated catalog contains references to secret environment variable names, never
// secret values. Core runs as an unprivileged container user and must be able to traverse
// the bind-mounted host directories and read the projections.
target := installation.GeneratedDirectory()
parent := filepath.Dir(target)
if err := os.MkdirAll(parent, 0o755); err != nil {
return fmt.Errorf("create model projection parent: %w", err)
}
candidate, err := os.MkdirTemp(parent, ".model-projections-candidate-*")
if err != nil {
return fmt.Errorf("create model projection candidate: %w", err)
}
defer func() { _ = os.RemoveAll(candidate) }()
if err := writeProjectionCandidate(candidate, artifacts); err != nil {
return err
}
info, statErr := os.Lstat(target)
if os.IsNotExist(statErr) {
if err := renameProjectionDirectory(candidate, target); err != nil {
return fmt.Errorf("publish model projection generation: %w", err)
}
return nil
}
if statErr != nil {
return fmt.Errorf("inspect current model projection generation: %w", statErr)
}
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("current model projection path is not a regular directory")
}
previous, err := absentTemporaryPath(parent)
if err != nil {
return fmt.Errorf("reserve previous model projection generation: %w", err)
}
if err := renameProjectionDirectory(target, previous); err != nil {
return fmt.Errorf("prepare model projection generation replacement: %w", err)
}
if err := renameProjectionDirectory(candidate, target); err != nil {
if restoreErr := renameProjectionDirectory(previous, target); restoreErr != nil {
return fmt.Errorf("publish model projection generation: %v; restore previous generation: %w", err, restoreErr)
}
return fmt.Errorf("publish model projection generation: %w", err)
}
_ = os.RemoveAll(previous)
return nil
}
func writeProjectionCandidate(directory string, artifacts map[string][]byte) error {
if err := os.Chmod(directory, 0o755); err != nil {
return fmt.Errorf("protect model projection candidate: %w", err)
}
for _, relative := range sortedArtifactPaths(artifacts) {
destination := filepath.Join(directory, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
return fmt.Errorf("create model projection directory: %w", err)
}
if err := os.Chmod(filepath.Dir(destination), 0o755); err != nil {
return fmt.Errorf("protect model projection directory: %w", err)
}
temporary, err := os.CreateTemp(filepath.Dir(destination), ".projection-*")
if err != nil {
return fmt.Errorf("create model projection candidate: %w", err)
}
temporaryName := temporary.Name()
published := false
defer func() {
if !published {
_ = os.Remove(temporaryName)
}
}()
if err := temporary.Chmod(0o644); err == nil {
_, err = temporary.Write(artifacts[relative])
}
if closeErr := temporary.Close(); err == nil {
err = closeErr
}
if err != nil {
if err := os.WriteFile(destination, artifacts[relative], 0o644); err != nil {
return fmt.Errorf("write model projection candidate: %w", err)
}
if err := os.Rename(temporaryName, destination); err != nil {
return fmt.Errorf("publish model projection: %w", err)
}
published = true
}
return nil
}
func absentTemporaryPath(parent string) (string, error) {
path, err := os.MkdirTemp(parent, ".model-projections-previous-*")
if err != nil {
return "", err
}
if err := os.Remove(path); err != nil {
return "", err
}
return path, nil
}
// Check returns relative artifact names whose current bytes differ from the catalog projection.
func Check(installation config.Installation) ([]string, error) {
artifacts, err := Render(installation)
@@ -2,6 +2,7 @@ package modelprojection
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
@@ -41,9 +42,12 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T)
t.Fatalf("Pi models projection must contain only custom providers:\n%s", piModels)
}
settings := string(first[PiSettingsFile])
if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen"`) {
if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen2.5:7b"`) {
t.Fatalf("Pi settings projection missing enabled session models:\n%s", settings)
}
if !strings.Contains(piModels, `"id": "qwen2.5:7b"`) || !strings.Contains(catalog, `"id": "local/qwen"`) {
t.Fatalf("Pi projection must map canonical catalog keys to upstream model IDs")
}
compose := string(first[ComposeFile])
for _, expected := range []string{"THT_MODEL_CATALOG_FILE", "THT_DEFAULT_SESSION_MODEL", "zai/glm-5.3", "THT_INTERNAL_EMBEDDING_ID", "THT_INTERNAL_EMBEDDING_MODEL", "OLLAMA_MODEL", installation.GeneratedModelCatalogPath()} {
if !strings.Contains(compose, expected) {
@@ -52,6 +56,46 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T)
}
}
func TestGenerateRestoresWholePreviousGenerationWhenPublishFails(t *testing.T) {
installation := projectionFixture(t)
if err := Generate(installation); err != nil {
t.Fatalf("Generate() initial error = %v", err)
}
before := make(map[string][]byte)
for _, path := range []string{CatalogFile, PiModelsFile, PiSettingsFile, ComposeFile} {
contents, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
if err != nil {
t.Fatal(err)
}
before[path] = contents
}
provider := installation.ModelCatalog.Providers["zai"]
model := provider.Models["glm-5.3"]
model.Label = "Changed label"
provider.Models["glm-5.3"] = model
installation.ModelCatalog.Providers["zai"] = provider
originalRename := renameProjectionDirectory
t.Cleanup(func() { renameProjectionDirectory = originalRename })
renames := 0
renameProjectionDirectory = func(oldPath, newPath string) error {
renames++
if renames == 2 {
return errors.New("injected publish failure")
}
return os.Rename(oldPath, newPath)
}
if err := Generate(installation); err == nil {
t.Fatal("Generate() error = nil, want injected publish failure")
}
for path, expected := range before {
actual, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
if err != nil || !bytes.Equal(actual, expected) {
t.Fatalf("artifact %q changed after failed generation: err=%v", path, err)
}
}
}
func TestGeneratePublishesAllArtifactsAndCheckDetectsDrift(t *testing.T) {
installation := projectionFixture(t)
if err := Generate(installation); err != nil {
@@ -84,7 +128,7 @@ func projectionFixture(t *testing.T) config.Installation {
Providers: map[string]config.ModelProvider{
"deepseek": {Authentication: config.ModelAuthentication{Mode: "pi_auth"}, Session: &config.ModelSessionAdapter{Mode: "pi_builtin"}, Models: map[string]config.CatalogModel{"deepseek-v4-pro": {Session: &config.SessionModel{}}}},
"zai": {Endpoint: &config.ModelEndpoint{BaseURL: "https://api.z.ai/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, Models: map[string]config.CatalogModel{"glm-5.3": {Label: "GLM-5.3", Session: &config.SessionModel{Reasoning: true, ContextWindow: 200000, MaxTokens: 131072}}}},
"local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}},
"local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {UpstreamModel: "qwen2.5:7b", Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}},
},
},
}