fix: harden model catalog projections

This commit is contained in:
Codex
2026-09-02 19:25:01 +02:00
parent ce4c31a6fb
commit a6a5bf2036
38 changed files with 573 additions and 83 deletions
+1
View File
@@ -268,6 +268,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
); );
const listModels = deps?.listModels ?? createPiModelLister(config, { const listModels = deps?.listModels ?? createPiModelLister(config, {
modelCatalog: runtimeModelCatalog,
warn: (detail) => app.log.warn( warn: (detail) => app.log.warn(
{ component: "pi-model-list", detail }, { component: "pi-model-list", detail },
"Pi enabled-model configuration warning", "Pi enabled-model configuration warning",
@@ -3,16 +3,17 @@ import type { CatalogDatabase } from "../repository.js";
const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$"; const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$";
const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$"; const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$";
const historicalOrCanonicalModelPattern = `(${legacyModelPattern})|(${canonicalModelPattern})`;
export async function up(db: Kysely<CatalogDatabase>): Promise<void> { export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
await sql.raw(`alter table description_generation_runs await sql.raw(`alter table description_generation_runs
drop constraint description_generation_runs_model_id_check, drop constraint description_generation_runs_model_id_check,
add constraint description_generation_runs_model_id_check add constraint description_generation_runs_model_id_check
check (model_id ~ '${canonicalModelPattern}')`).execute(db); check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
await sql.raw(`alter table sensitive_data_suggestion_runs await sql.raw(`alter table sensitive_data_suggestion_runs
drop constraint sensitive_data_suggestion_runs_model_id_check, drop constraint sensitive_data_suggestion_runs_model_id_check,
add constraint sensitive_data_suggestion_runs_model_id_check add constraint sensitive_data_suggestion_runs_model_id_check
check (model_id ~ '${canonicalModelPattern}')`).execute(db); check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
} }
export async function down(db: Kysely<CatalogDatabase>): Promise<void> { export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+17 -1
View File
@@ -10,6 +10,7 @@ import {
readConfiguredPiAgentFile, readConfiguredPiAgentFile,
validateDeclarativePiConfig, validateDeclarativePiConfig,
} from "./managed-config.js"; } from "./managed-config.js";
import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
export interface PiModel { export interface PiModel {
provider: string; provider: string;
@@ -30,6 +31,7 @@ interface Opts {
nowMs?: () => number; nowMs?: () => number;
loadEnabledModels?: () => PiEnabledModelsResult; loadEnabledModels?: () => PiEnabledModelsResult;
readModelsStore?: () => string | undefined; readModelsStore?: () => string | undefined;
modelCatalog?: RuntimeModelCatalog;
warn?: (message: string) => void; warn?: (message: string) => void;
} }
@@ -82,9 +84,23 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModels
const byCompositeId = new Map( const byCompositeId = new Map(
available.map((model) => [`${model.provider}/${model.id}`, model]), available.map((model) => [`${model.provider}/${model.id}`, model]),
); );
const catalogByPiId = new Map(
(opts.modelCatalog?.sessionModels() ?? []).map((model) => [
`${model.provider}/${model.upstreamModel}`,
model,
]),
);
const models = enabled.ids.flatMap((id) => { const models = enabled.ids.flatMap((id) => {
const model = byCompositeId.get(id); const model = byCompositeId.get(id);
return model ? [model] : []; if (!model) return [];
const catalogModel = catalogByPiId.get(id);
return [catalogModel ? {
...model,
provider: catalogModel.provider,
id: catalogModel.model,
name: catalogModel.label,
reasoning: catalogModel.session?.reasoning ?? model.reasoning,
} : model];
}); });
if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available"); if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available");
cache = { at: now(), models }; cache = { at: now(), models };
+4 -1
View File
@@ -260,8 +260,11 @@ export class PiProcessManager {
const thinking = o.thinking ?? this.cfg.defaults.thinking; const thinking = o.thinking ?? this.cfg.defaults.thinking;
if (provider && model) { if (provider && model) {
const upstreamModel = this.modelCatalog.sessionModels()
.find((entry) => entry.provider === provider && entry.model === model)
?.upstreamModel ?? model;
const response = await rt.rpc.request( const response = await rt.rpc.request(
{ type: "set_model", provider, modelId: model } as object & { type: string }, { type: "set_model", provider, modelId: upstreamModel } as object & { type: string },
); );
rt.bridge.setContextWindow(response?.data?.contextWindow); rt.bridge.setContextWindow(response?.data?.contextWindow);
} }
+2 -1
View File
@@ -79,6 +79,7 @@ export function createPiProviderSmoke(
|| catalog.defaultSession !== null; || catalog.defaultSession !== null;
const catalogModel = catalog.sessionModels() const catalogModel = catalog.sessionModels()
.find((entry) => entry.provider === canonicalProvider && entry.model === model); .find((entry) => entry.provider === canonicalProvider && entry.model === model);
const upstreamModel = catalogModel?.upstreamModel ?? model;
const credentialName = catalogModel?.authentication.mode === "secret_env" const credentialName = catalogModel?.authentication.mode === "secret_env"
? catalogModel.authentication.apiKeyEnv ? catalogModel.authentication.apiKeyEnv
: undefined; : undefined;
@@ -130,7 +131,7 @@ export function createPiProviderSmoke(
const capabilityGuard = failOnUnexpectedCapabilities(rpc); const capabilityGuard = failOnUnexpectedCapabilities(rpc);
const turn = async (): Promise<void> => { const turn = async (): Promise<void> => {
requireSuccessfulResponse(await rpc.request({ requireSuccessfulResponse(await rpc.request({
type: "set_model", provider: canonicalProvider, modelId: model, type: "set_model", provider: canonicalProvider, modelId: upstreamModel,
} as object & { type: string })); } as object & { type: string }));
requireSuccessfulResponse(await rpc.request({ requireSuccessfulResponse(await rpc.request({
type: "set_thinking_level", level: reasoning, type: "set_thinking_level", level: reasoning,
+1 -1
View File
@@ -140,7 +140,7 @@ function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbed
} }
const model = requireString(embeddings, "model"); const model = requireString(embeddings, "model");
return { return {
id: `ollama/${model}`, id: optionalString(embeddings, "id") ?? `ollama/${model}`,
model, model,
dimensions: requireNumber(embeddings, "dimensions"), dimensions: requireNumber(embeddings, "dimensions"),
}; };
@@ -274,6 +274,8 @@ export function renderRuntimeConfig(
}, },
embedding: { embedding: {
provider: "ollama_internal", provider: "ollama_internal",
id: semanticRuntime.internalEmbeddingId
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
model: semanticRuntime.internalEmbeddingModel, model: semanticRuntime.internalEmbeddingModel,
dimensions: semanticRuntime.internalEmbeddingDimensions, dimensions: semanticRuntime.internalEmbeddingDimensions,
}, },
@@ -287,6 +289,8 @@ export function renderRuntimeConfig(
embeddings: { embeddings: {
provider: "ollama_internal", provider: "ollama_internal",
base_url: semanticRuntime.internalEmbeddingUrl, base_url: semanticRuntime.internalEmbeddingUrl,
id: semanticRuntime.internalEmbeddingId
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
model: semanticRuntime.internalEmbeddingModel, model: semanticRuntime.internalEmbeddingModel,
dimensions: semanticRuntime.internalEmbeddingDimensions, dimensions: semanticRuntime.internalEmbeddingDimensions,
}, },
@@ -1,4 +1,5 @@
import { spawnSync } from "node:child_process"; import { spawnSync } from "node:child_process";
import { randomUUID } from "node:crypto";
import { PostgreSqlContainer } from "@testcontainers/postgresql"; import { PostgreSqlContainer } from "@testcontainers/postgresql";
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely"; import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
import { Pool } from "pg"; import { Pool } from "pg";
@@ -33,7 +34,30 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
await upDescriptionGeneration(db); await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db); await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db); await upAiTokenUsage(db);
const historicalDatabaseId = randomUUID();
await db.insertInto("workspaceDatabases").values({
id: historicalDatabaseId,
workspaceId: "migration-history",
engine: "postgres",
databaseName: "warehouse",
schemaName: "public",
}).execute();
await db.insertInto("descriptionGenerationRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
modelId: "openai-mini", language: "en", status: "completed", total: 1,
processed: 1, generated: 1,
}).execute();
await db.insertInto("sensitiveDataSuggestionRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
modelId: "openai-mini", status: "completed", total: 1,
suggestedSensitive: 1,
}).execute();
await upCanonicalModelIds(db); await upCanonicalModelIds(db);
await expect(db.insertInto("descriptionGenerationRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1,
processed: 1, generated: 1,
}).execute()).resolves.toBeDefined();
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db); await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
await upRuntimeSequencePrivileges(db); await upRuntimeSequencePrivileges(db);
const sequencePrivilege = await sql<{ allowed: boolean }>` const sequencePrivilege = await sql<{ allowed: boolean }>`
+31
View File
@@ -6,6 +6,7 @@ import { join } from "node:path";
import path from "node:path"; import path from "node:path";
import { createPiModelLister } from "../src/pi/list-models.js"; import { createPiModelLister } from "../src/pi/list-models.js";
import { loadConfig } from "../src/config.js"; import { loadConfig } from "../src/config.js";
import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs"); const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
@@ -44,6 +45,36 @@ test("createPiModelLister returns mapped PiModel[] from get_available_models", a
} }
}); });
test("catalog listing translates upstream Pi IDs back to canonical model keys", async () => {
const script = scriptWith([
{ provider: "local", id: "qwen2.5:7b", name: "Upstream label", reasoning: false },
]);
const model: RuntimeModel = {
id: "local/qwen", provider: "local", model: "qwen", label: "Catalog Qwen",
upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 },
};
const modelCatalog: RuntimeModelCatalog = {
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
};
try {
const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
...noManagedModels,
modelCatalog,
loadEnabledModels: enabled("local/qwen2.5:7b"),
spawnFn: () => spawn("node", [FAKE, script]) as any,
});
await expect(lister()).resolves.toEqual([{
provider: "local", id: "qwen", name: "Catalog Qwen", reasoning: true,
}]);
} finally {
rmSync(path.dirname(script), { recursive: true, force: true });
}
});
test("createPiModelLister caches within ttl (spawns once for two calls)", async () => { test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]); const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
try { try {
+48
View File
@@ -718,6 +718,54 @@ test.each([["OpenAI", "openai"], ["gemini", "google"]])(
}, },
); );
test("set_model translates a canonical catalog key to its upstream Pi model ID", async () => {
const root = mkdtempSync(path.join(tmpdir(), "thothii-upstream-model-"));
const agentDir = path.join(root, "agent");
mkdirSync(agentDir, { mode: 0o700 });
writeFileSync(path.join(agentDir, "models.json"), JSON.stringify({
providers: {
local: {
baseUrl: "http://ollama:11434/v1", apiKey: "local",
models: [{ id: "qwen2.5:7b" }],
},
},
}), { mode: 0o600 });
vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
const child = recordingChild();
child.stderr.resume = () => {};
child.stdin.write = (data: unknown) => {
const request = JSON.parse(String(data));
child._writes.push(String(data));
if (request.id) {
queueMicrotask(() => child.stdout.emit("data", `${JSON.stringify({
type: "response", id: request.id, success: true,
})}\n`));
}
return true;
};
const model: RuntimeModel = {
id: "local/qwen", provider: "local", model: "qwen", label: "Qwen",
upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
session: { reasoning: false, contextWindow: 32768, maxTokens: 8192 },
};
const modelCatalog: RuntimeModelCatalog = {
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
};
const mgr = new PiProcessManager(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
modelCatalog, authProviders: () => new Set(), spawnFn: () => child as any,
});
try {
await mgr.spawnFor("upstream-model", { provider: "local", model: "qwen" });
expect(child._writes.join("")).toContain('"modelId":"qwen2.5:7b"');
} finally {
mgr.teardown("upstream-model");
vi.unstubAllEnvs();
rmSync(root, { recursive: true, force: true });
}
});
test.each(["installation-local", "private-compatible"])( test.each(["installation-local", "private-compatible"])(
"provider %s configured with a literal apiKey spawns without a managed key", "provider %s configured with a literal apiKey spawns without a managed key",
async (provider) => { async (provider) => {
+12 -1
View File
@@ -300,11 +300,22 @@ test("provider smoke makes one configured request from an isolated no-capability
}); });
} }
}); });
const smokeModel: RuntimeModel = {
id: "zai/catalog-glm", provider: "zai", model: "catalog-glm", label: "GLM",
upstreamModel: "glm-5.2", authentication: { mode: "pi_auth" },
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true },
};
const smokeCatalog: RuntimeModelCatalog = {
defaultSession: smokeModel.id, defaultMetadataGeneration: null, embedding: null,
sessionModels: () => [smokeModel], metadataModels: () => [],
hasSession: (id) => id === smokeModel.id,
};
const smoke = createPiProviderSmoke(loadConfig({ const smoke = createPiProviderSmoke(loadConfig({
THT_HARNESS_DIR: "/app/harness", THT_HARNESS_DIR: "/app/harness",
PI_BIN: "/usr/local/bin/pi", PI_BIN: "/usr/local/bin/pi",
THT_DATA_ROOT: "/mounted-workflow-state", THT_DATA_ROOT: "/mounted-workflow-state",
}), { }), {
modelCatalog: smokeCatalog,
spawnFn: (...args) => { spawnFn: (...args) => {
spawns.push(args); spawns.push(args);
expect(args[2].cwd).not.toBe("/app/harness"); expect(args[2].cwd).not.toBe("/app/harness");
@@ -324,7 +335,7 @@ test("provider smoke makes one configured request from an isolated no-capability
}); });
await expect(smoke({ await expect(smoke({
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750, provider: "zai", model: "catalog-glm", reasoning: "medium", timeoutMs: 750,
})).resolves.toBeUndefined(); })).resolves.toBeUndefined();
expect(spawns).toHaveLength(1); expect(spawns).toHaveLength(1);
expect(spawns[0][0]).toBe("/usr/local/bin/pi"); expect(spawns[0][0]).toBe("/usr/local/bin/pi");
@@ -73,6 +73,7 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" }, vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
embeddings: { embeddings: {
provider: "ollama_internal", base_url: "http://embedding:11434", provider: "ollama_internal", base_url: "http://embedding:11434",
id: "ollama/qwen3-embedding:0.6b",
model: "qwen3-embedding:0.6b", dimensions: 1024, model: "qwen3-embedding:0.6b", dimensions: 1024,
}, },
}, },
@@ -35,7 +35,6 @@ resources:
base_url: http://qdrant:6333 base_url: http://qdrant:6333
collection: server-sessions collection: server-sessions
embeddings: embeddings:
# id, model, and dimensions are injected from the Installation Model Catalog.
provider: ollama_internal provider: ollama_internal
base_url: http://embedding:11434 base_url: http://embedding:11434
model: qwen3-embedding:0.6b
dimensions: 1024
+9 -7
View File
@@ -100,7 +100,7 @@ generated/
├── pi/ ├── pi/
│ ├── models.json │ ├── models.json
│ └── settings.json │ └── settings.json
└── compose.model-catalog.yaml └── compose.models.yaml
``` ```
The normalized catalog is consumed by the backend. The Pi files and Compose override are boundary The normalized catalog is consumed by the backend. The Pi files and Compose override are boundary
@@ -117,17 +117,19 @@ tht --installation "$INSTALLATION" start
tht --installation "$INSTALLATION" doctor tht --installation "$INSTALLATION" doctor
``` ```
After editing `modelCatalog` or provider credentials, reload the current Pi image. Restart validates After editing `modelCatalog` or provider credentials, apply the complete runtime projection with
the YAML and regenerates projections before recreating `core`: the normal installation lifecycle, then run the Pi checks:
```bash ```bash
tht --installation "$INSTALLATION" pi restart --yes --drain tht --installation "$INSTALLATION" start
tht --installation "$INSTALLATION" pi doctor tht --installation "$INSTALLATION" pi doctor
tht --installation "$INSTALLATION" pi test tht --installation "$INSTALLATION" pi test
``` ```
`tht pi update` changes the Pi version; it is not the configuration command. There is no `tht pi restart`, `tht pi update`, and `tht pi rollback` refuse to run while generated model
`tht pi configure` and no separate apply command. projections differ from `modelCatalog`: those commands recreate only `core`, so they must never
partially apply an embedding change. `tht pi update` changes the Pi version; it is not the
configuration command. There is no `tht pi configure` and no separate apply command.
## Migrating a legacy installation ## Migrating a legacy installation
@@ -136,7 +138,7 @@ files, but never modifies them. Supply the facts that cannot be inferred safely
candidate: candidate:
```bash ```bash
tht --installation /absolute/path/legacy-installation.yaml installation migrate \ tht --installation /absolute/path/legacy/thothii-installation.yaml installation migrate \
--output /absolute/path/thothii-installation.v2.yaml \ --output /absolute/path/thothii-installation.v2.yaml \
--session-default zai/glm-5.3 \ --session-default zai/glm-5.3 \
--embedding-id ollama/qwen3-embedding:0.6b \ --embedding-id ollama/qwen3-embedding:0.6b \
@@ -166,6 +166,8 @@ Generation is deterministic and published only after every candidate artifact va
generation aborts start before Compose is invoked. `tht doctor` recomputes expected bytes and reports generation aborts start before Compose is invoked. `tht doctor` recomputes expected bytes and reports
differences; no digest manifest or separate apply command exists. When projection bytes change, differences; no digest manifest or separate apply command exists. When projection bytes change,
`tht start` recreates the affected services so they cannot continue with an older bind mount. `tht start` recreates the affected services so they cannot continue with an older bind mount.
Pi-only restart, update, and rollback operations reject projection drift and direct the operator to
`tht start`, because applying only the core-facing files could leave embedding services stale.
Generated projections are not backed up. Restore validates the canonical installation descriptor, Generated projections are not backed up. Restore validates the canonical installation descriptor,
regenerates every projection, and only then starts services. Base Compose files and `operator.env` regenerates every projection, and only then starts services. Base Compose files and `operator.env`
+57 -5
View File
@@ -214,7 +214,7 @@ embeddings: {provider: ollama_internal, base_url: http://embedding:11434, model:
assert cfg.vectors.writer.api_key == "writer" assert cfg.vectors.writer.api_key == "writer"
def test_accepts_only_internal_ollama_embedding_contract(tmp_path): def test_accepts_catalog_selected_internal_ollama_embedding_contract(tmp_path):
workspace = tmp_path / "workspace.yaml" workspace = tmp_path / "workspace.yaml"
workspace.write_text( workspace.write_text(
""" """
@@ -225,8 +225,9 @@ resources:
embeddings: embeddings:
provider: ollama_internal provider: ollama_internal
base_url: http://embedding:11434 base_url: http://embedding:11434
model: qwen3-embedding:0.6b id: ollama/bge-m3
dimensions: 1024 model: bge-m3
dimensions: 1536
""" """
) )
@@ -234,8 +235,59 @@ resources:
assert cfg.embeddings.provider == "ollama_internal" assert cfg.embeddings.provider == "ollama_internal"
assert cfg.embeddings.base_url == "http://embedding:11434" assert cfg.embeddings.base_url == "http://embedding:11434"
assert cfg.embeddings.model == "qwen3-embedding:0.6b" assert cfg.embeddings.id == "ollama/bge-m3"
assert cfg.embeddings.dim == 1024 assert cfg.embeddings.model == "bge-m3"
assert cfg.embeddings.dim == 1536
def test_installation_embedding_projection_completes_model_free_runtime_source(
tmp_path, monkeypatch,
):
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536")
workspace = tmp_path / "workspace.yaml"
workspace.write_text(
"""
dwh:
type: postgres_direct
connection: {database: analytics, schema: mart, user: reader, password: secret}
resources:
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
"""
)
cfg = load_config(workspace)
assert cfg.embeddings.id == "ollama/bge-m3"
assert cfg.embeddings.model == "bge-m3"
assert cfg.embeddings.dim == 1536
def test_workspace_embedding_values_cannot_override_installation_projection(
tmp_path, monkeypatch,
):
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3")
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536")
workspace = tmp_path / "workspace.yaml"
workspace.write_text(
"""
dwh:
type: postgres_direct
connection: {database: analytics, schema: mart, user: reader, password: secret}
resources:
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
model: workspace-owned-model
"""
)
with pytest.raises(ConfigError, match="proprietà dell'installazione|diverge"):
load_config(workspace)
def test_accepts_internal_qdrant_resource_contract(tmp_path): def test_accepts_internal_qdrant_resource_contract(tmp_path):
+21
View File
@@ -40,6 +40,7 @@ def test_manifest_contains_provenance_without_credentials():
manifest_id="manifest:abc", manifest_id="manifest:abc",
created_at=datetime(2026, 7, 12, tzinfo=UTC), created_at=datetime(2026, 7, 12, tzinfo=UTC),
pipeline_version="evidence-v1", pipeline_version="evidence-v1",
embedding_id="ollama/nomic-embed-text",
embedding_model="nomic-embed-text", embedding_model="nomic-embed-text",
embedding_dimensions=768, embedding_dimensions=768,
documents=[document()], documents=[document()],
@@ -52,6 +53,7 @@ def test_manifest_contains_provenance_without_credentials():
assert "etag:abc" in payload assert "etag:abc" in payload
assert "evidence-v1" in payload assert "evidence-v1" in payload
assert "nomic-embed-text" in payload assert "nomic-embed-text" in payload
assert '"embedding_id":"ollama/nomic-embed-text"' in payload
assert "api_key" not in payload assert "api_key" not in payload
@@ -93,6 +95,25 @@ def test_manifest_validates_embedding_compatibility_fields():
) )
def test_schema_v1_manifest_without_canonical_embedding_id_remains_readable():
manifest = CorpusManifest.model_validate({
"schema_version": 1,
"embedding_model": "legacy-model",
"embedding_dimensions": 768,
})
assert manifest.embedding_id is None
def test_schema_v2_embedding_generation_requires_canonical_id():
with pytest.raises(ValidationError, match="embedding_id"):
CorpusManifest(
schema_version=2,
embedding_model="model-v2",
embedding_dimensions=768,
)
def test_canonical_metadata_rejects_secrets_and_non_json_values(): def test_canonical_metadata_rejects_secrets_and_non_json_values():
with pytest.raises(ValidationError, match="credential-like"): with pytest.raises(ValidationError, match="credential-like"):
CanonicalDocument.model_validate( CanonicalDocument.model_validate(
+21 -1
View File
@@ -105,11 +105,13 @@ def item(name, fingerprint):
) )
def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None, def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a",
embedding_id=None, policy=None,
retain=3, candidate_evaluator=None): retain=3, candidate_evaluator=None):
return CorpusPipeline( return CorpusPipeline(
store=CorpusStore(tmp_path / "corpus"), sources=[source], store=CorpusStore(tmp_path / "corpus"), sources=[source],
embedder=embedder or Embedder(), vector_store=vectors or Vectors(), embedder=embedder or Embedder(), vector_store=vectors or Vectors(),
embedding_id=embedding_id,
embedding_model=model, embedding_dimensions=3, embedding_model=model, embedding_dimensions=3,
chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100), chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100),
pipeline_version="evidence-v1", pipeline_version="evidence-v1",
@@ -875,6 +877,24 @@ def test_model_or_chunk_policy_change_forces_full_rebuild(tmp_path):
assert source.acquire_calls == ["fs:one"] assert source.acquire_calls == ["fs:one"]
def test_canonical_embedding_id_change_forces_full_rebuild_and_is_persisted(tmp_path):
one = item("one", "a")
first = pipeline(
tmp_path, Source([(one, "hello")]), model="same-upstream",
embedding_id="ollama/catalog-a",
).run()
assert first.manifest.embedding_id == "ollama/catalog-a"
source = Source([(one, "hello")])
changed = pipeline(
tmp_path, source, model="same-upstream", embedding_id="ollama/catalog-b",
).run()
assert changed.changed == ("fs:one",)
assert changed.manifest.embedding_id == "ollama/catalog-b"
assert source.acquire_calls == ["fs:one"]
def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path): def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path):
one = item("one", "a") one = item("one", "a")
good = pipeline(tmp_path, Source([(one, "old")])) good = pipeline(tmp_path, Source([(one, "old")]))
+16 -2
View File
@@ -21,7 +21,8 @@ def _write_cfg(tmp_path, raw):
return cfg return cfg
def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid", collection="psd", model="qwen3-embedding:0.6b"): def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid",
collection="psd", model="qwen3-embedding:0.6b", dimensions=1024):
return { return {
"schemaVersion": 1, "schemaVersion": 1,
"workspace": {"schema_version": 3, "id": "psd", "name": "PSD", "language": "it"}, "workspace": {"schema_version": 3, "id": "psd", "name": "PSD", "language": "it"},
@@ -34,7 +35,10 @@ def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.inval
"connection": {"host": "h", "port": 5432, "database": "warehouse", "schema": "dw", "user": "reader", "password": "secret"}, "connection": {"host": "h", "port": 5432, "database": "warehouse", "schema": "dw", "user": "reader", "password": "secret"},
}, },
"vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": collection, "collection_lifecycle": "self_heal"}, "vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": collection, "collection_lifecycle": "self_heal"},
"embeddings": {"provider": "ollama_internal", "base_url": "http://embedding:11434", "model": model, "dimensions": 1024}, "embeddings": {
"provider": "ollama_internal", "base_url": "http://embedding:11434",
"id": f"ollama/{model}", "model": model, "dimensions": dimensions,
},
"roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")}, "roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")},
"paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")}, "paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")},
} }
@@ -52,6 +56,16 @@ def test_canonical_json_is_deterministic_and_key_ordered(cfg):
assert keys == ["schemaVersion", "dwh", "vector", "embedding", "roots"] assert keys == ["schemaVersion", "dwh", "vector", "embedding", "roots"]
def test_catalog_embedding_identity_and_dimensions_drive_effective_config(tmp_path):
cfg = _write_cfg(tmp_path, _cfg(tmp_path, model="bge-m3", dimensions=1536))
document = __import__("json").loads(canonical_effective_config_json(cfg))
assert document["embedding"] == {
"id": "ollama/bge-m3", "model": "bge-m3", "dimensions": 1536,
}
assert document["vector"]["dimensions"] == 1536
def test_canonical_excludes_credentials_and_evidence(cfg): def test_canonical_excludes_credentials_and_evidence(cfg):
doc = canonical_effective_config_json(cfg) doc = canonical_effective_config_json(cfg)
assert "secret" not in doc assert "secret" not in doc
@@ -149,6 +149,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk
"sources": [object()], "sources": [object()],
"embedder": object(), "embedder": object(),
"vector_store": object(), "vector_store": object(),
"embedding_id": "ollama/model",
"embedding_model": "model", "embedding_model": "model",
"embedding_dimensions": 3, "embedding_dimensions": 3,
"chunk_policy": object(), "chunk_policy": object(),
+2
View File
@@ -10,6 +10,8 @@ from tht.config import EmbeddingsConfig
def _cfg(**kw): def _cfg(**kw):
kw.setdefault("model", "nomic-embed-text-v2-moe")
kw.setdefault("dim", 768)
emb = EmbeddingsConfig(base_url="http://localhost:11434", **kw) emb = EmbeddingsConfig(base_url="http://localhost:11434", **kw)
return SimpleNamespace(embeddings=emb) return SimpleNamespace(embeddings=emb)
+3
View File
@@ -284,6 +284,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
sources, sources,
embedder, embedder,
vector_store, vector_store,
embedding_id,
embedding_model, embedding_model,
embedding_dimensions, embedding_dimensions,
chunk_policy, chunk_policy,
@@ -293,6 +294,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
candidate_evaluator, candidate_evaluator,
): ):
calls["init"] = { calls["init"] = {
"embedding_id": embedding_id,
"embedding_model": embedding_model, "embedding_model": embedding_model,
"embedding_dimensions": embedding_dimensions, "embedding_dimensions": embedding_dimensions,
"pipeline_version": pipeline_version, "pipeline_version": pipeline_version,
@@ -314,6 +316,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
command.run_from_config(config) command.run_from_config(config)
assert calls["init"]["embedding_id"] == "ollama/qwen3-embedding:0.6b"
assert calls["init"]["sparse_language"] == "english" assert calls["init"]["sparse_language"] == "english"
assert calls["init"]["candidate_evaluator"] is None assert calls["init"]["candidate_evaluator"] is None
assert calls["run_as_job"]["workspace_id"] == "psd-clinical" assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
+2
View File
@@ -187,6 +187,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence), store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
embedder=embedder, embedder=embedder,
vector_store=vector_store, vector_store=vector_store,
embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}",
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1", pipeline_version="evidence-v1",
@@ -222,6 +223,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
pipeline = build_preprocessing_pipeline( pipeline = build_preprocessing_pipeline(
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence), store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True), embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True),
embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}",
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1", pipeline_version="evidence-v1",
+84 -13
View File
@@ -60,8 +60,12 @@ def canonical_effective_config_document(cfg) -> dict:
return { return {
"schemaVersion": 1, "schemaVersion": 1,
"dwh": dwh, "dwh": dwh,
"vector": {"collection": collection, "dimensions": 1024, "distance": "cosine"}, "vector": {"collection": collection, "dimensions": int(embed_dim), "distance": "cosine"},
"embedding": {"model": model, "dimensions": int(embed_dim)}, "embedding": {
"id": getattr(embeddings, "id", None) or f"ollama/{model}",
"model": model,
"dimensions": int(embed_dim),
},
"roots": { "roots": {
"artifacts": str(getattr(cfg.paths, "artifacts", Path("artifacts"))), "artifacts": str(getattr(cfg.paths, "artifacts", Path("artifacts"))),
"indexes": str(getattr(cfg.paths, "indexes", Path("indexes"))), "indexes": str(getattr(cfg.paths, "indexes", Path("indexes"))),
@@ -499,8 +503,9 @@ class EvidenceSourcesConfig(BaseModel):
class EmbeddingsConfig(BaseModel): class EmbeddingsConfig(BaseModel):
provider: str = "ollama_internal" provider: str = "ollama_internal"
base_url: str base_url: str
model: str = "nomic-embed-text-v2-moe" id: str | None = None
dim: int = Field(default=768, alias="dimensions") model: str
dim: int = Field(alias="dimensions")
batch_size: int = 16 batch_size: int = 16
timeout: int = 300 timeout: int = 300
connect_timeout: int = 5 connect_timeout: int = 5
@@ -642,6 +647,7 @@ def load_config(path: Path) -> Config:
if not isinstance(raw, dict): if not isinstance(raw, dict):
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}") raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
expanded = _resolve_secret_files(_resolve_evidence_secret_files(_expand_env(raw))) expanded = _resolve_secret_files(_resolve_evidence_secret_files(_expand_env(raw)))
_apply_installation_embedding_projection(expanded, path)
_validate_internal_embedding_contract(expanded, path) _validate_internal_embedding_contract(expanded, path)
_validate_internal_vector_contract(expanded, path) _validate_internal_vector_contract(expanded, path)
translated, used_legacy = translate_legacy_config(expanded) translated, used_legacy = translate_legacy_config(expanded)
@@ -710,6 +716,56 @@ def load_config(path: Path) -> Config:
return cfg return cfg
def _apply_installation_embedding_projection(raw: dict[str, Any], path: Path) -> None:
projected = {
"id": os.environ.get("THT_INTERNAL_EMBEDDING_ID"),
"model": os.environ.get("THT_INTERNAL_EMBEDDING_MODEL"),
"dimensions": os.environ.get("THT_INTERNAL_EMBEDDING_DIMENSIONS"),
}
if all(value is None for value in projected.values()):
return
if any(value is None for value in projected.values()):
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"la proiezione embedding dell'installazione è incompleta"
)
embedding_id = projected["id"]
model = projected["model"]
try:
dimensions = int(projected["dimensions"] or "")
except ValueError:
dimensions = 0
if embedding_id != f"ollama/{model}" or dimensions <= 0:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"la proiezione embedding dell'installazione non è canonica"
)
sections: list[tuple[dict[str, Any], str]] = []
embeddings = raw.get("embeddings")
if isinstance(embeddings, dict):
sections.append((embeddings, "dim"))
resources = raw.get("resources")
if isinstance(resources, dict) and isinstance(resources.get("embeddings"), dict):
sections.append((resources["embeddings"], "dimensions"))
for section, dimension_key in sections:
declared = {
"id": section.get("id"),
"model": section.get("model"),
"dimensions": section.get(dimension_key),
}
expected = {"id": embedding_id, "model": model, "dimensions": dimensions}
for key, value in declared.items():
if value is not None and str(value) != str(expected[key]):
raise ConfigError(
f"Configurazione non valida in {path}:\n"
f"{key} è proprietà dell'installazione e diverge dalla proiezione attiva"
)
section["id"] = embedding_id
section["model"] = model
section[dimension_key] = dimensions
def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> None: def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> None:
resources = raw.get("resources") resources = raw.get("resources")
if not isinstance(resources, dict): if not isinstance(resources, dict):
@@ -720,9 +776,10 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
provider = embeddings.get("provider") provider = embeddings.get("provider")
model = embeddings.get("model") model = embeddings.get("model")
embedding_id = embeddings.get("id")
dimensions = embeddings.get("dimensions") dimensions = embeddings.get("dimensions")
base_url = embeddings.get("base_url") base_url = embeddings.get("base_url")
allowed = {"provider", "base_url", "model", "dimensions"} allowed = {"provider", "base_url", "id", "model", "dimensions"}
unexpected = sorted(set(embeddings) - allowed) unexpected = sorted(set(embeddings) - allowed)
if unexpected: if unexpected:
raise ConfigError( raise ConfigError(
@@ -734,15 +791,24 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
f"Configurazione non valida in {path}:\n" f"Configurazione non valida in {path}:\n"
"resources.embeddings.provider deve essere 'ollama_internal'" "resources.embeddings.provider deve essere 'ollama_internal'"
) )
if model != "qwen3-embedding:0.6b": if not isinstance(model, str) or not model:
raise ConfigError( raise ConfigError(
f"Configurazione non valida in {path}:\n" f"Configurazione non valida in {path}:\n"
"resources.embeddings.model deve essere 'qwen3-embedding:0.6b'" "resources.embeddings.model deve essere valorizzato"
) )
if dimensions != 1024: try:
parsed_dimensions = int(dimensions)
except (TypeError, ValueError):
parsed_dimensions = 0
if isinstance(dimensions, bool) or parsed_dimensions <= 0:
raise ConfigError( raise ConfigError(
f"Configurazione non valida in {path}:\n" f"Configurazione non valida in {path}:\n"
"resources.embeddings.dimensions deve essere 1024" "resources.embeddings.dimensions deve essere un intero positivo"
)
if embedding_id is not None and embedding_id != f"ollama/{model}":
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"resources.embeddings.id deve essere l'identità canonica ollama/<model>"
) )
if not _is_allowed_internal_embedding_url(base_url): if not _is_allowed_internal_embedding_url(base_url):
raise ConfigError( raise ConfigError(
@@ -805,15 +871,20 @@ def _validate_active_embeddings_config(
f"Configurazione non valida in {path}:\n" f"Configurazione non valida in {path}:\n"
"embeddings.provider deve essere 'ollama_internal'" "embeddings.provider deve essere 'ollama_internal'"
) )
if embeddings.model != "qwen3-embedding:0.6b": if not embeddings.model:
raise ConfigError( raise ConfigError(
f"Configurazione non valida in {path}:\n" f"Configurazione non valida in {path}:\n"
"embeddings.model deve essere 'qwen3-embedding:0.6b'" "embeddings.model deve essere valorizzato"
) )
if embeddings.dim != 1024: if embeddings.dim <= 0:
raise ConfigError( raise ConfigError(
f"Configurazione non valida in {path}:\n" f"Configurazione non valida in {path}:\n"
"embeddings.dim deve essere 1024" "embeddings.dim deve essere un intero positivo"
)
if embeddings.id is not None and embeddings.id != f"ollama/{embeddings.model}":
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"embeddings.id deve essere l'identità canonica ollama/<model>"
) )
if not _is_allowed_internal_embedding_url(embeddings.base_url): if not _is_allowed_internal_embedding_url(embeddings.base_url):
raise ConfigError( raise ConfigError(
+17
View File
@@ -19,6 +19,9 @@ from tht.evidence.contracts import (
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$") _NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") _SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
_EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$") _EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$")
_CANONICAL_MODEL_ID = re.compile(
r"^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$"
)
_EVIDENCE_METADATA_KEYS = frozenset({ _EVIDENCE_METADATA_KEYS = frozenset({
"evidence_id", "evidence_kind", "purposes", "scope", "language", "provenance", "evidence_id", "evidence_kind", "purposes", "scope", "language", "provenance",
}) })
@@ -37,6 +40,12 @@ def _validate_hash(value: str) -> str:
return value return value
def _validate_canonical_model_id(value: str) -> str:
if not _CANONICAL_MODEL_ID.fullmatch(value):
raise ValueError("model identifier must use canonical provider/model form")
return value
def _require_content_hash(content: str, content_hash: str) -> None: def _require_content_hash(content: str, content_hash: str) -> None:
expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}" expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
if content_hash != expected: if content_hash != expected:
@@ -146,6 +155,7 @@ class CorpusManifest(_WithMetadata):
manifest_id: str | None = None manifest_id: str | None = None
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
pipeline_version: str = Field(default="evidence-v1", min_length=1) pipeline_version: str = Field(default="evidence-v1", min_length=1)
embedding_id: str | None = None
embedding_model: str | None = None embedding_model: str | None = None
embedding_dimensions: int | None = Field(default=None, gt=0) embedding_dimensions: int | None = Field(default=None, gt=0)
vector_generation: str | None = None vector_generation: str | None = None
@@ -155,6 +165,9 @@ class CorpusManifest(_WithMetadata):
_manifest_id = field_validator("manifest_id")( _manifest_id = field_validator("manifest_id")(
lambda value: _validate_namespaced_id(value) if value is not None else None lambda value: _validate_namespaced_id(value) if value is not None else None
) )
_embedding_id = field_validator("embedding_id")(
lambda value: _validate_canonical_model_id(value) if value is not None else None
)
_vector_generation = field_validator("vector_generation")( _vector_generation = field_validator("vector_generation")(
lambda value: _validate_namespaced_id(value) if value is not None else None lambda value: _validate_namespaced_id(value) if value is not None else None
) )
@@ -164,6 +177,10 @@ class CorpusManifest(_WithMetadata):
def validate_generation(self) -> "CorpusManifest": def validate_generation(self) -> "CorpusManifest":
if (self.embedding_model is None) != (self.embedding_dimensions is None): if (self.embedding_model is None) != (self.embedding_dimensions is None):
raise ValueError("embedding_model and embedding_dimensions must be set together") raise ValueError("embedding_model and embedding_dimensions must be set together")
if self.embedding_id is not None and self.embedding_model is None:
raise ValueError("embedding_id requires embedding model and dimensions")
if self.schema_version >= 2 and self.embedding_model is not None and self.embedding_id is None:
raise ValueError("schema version 2 embedding generations require embedding_id")
if self.vector_generation is not None and self.embedding_model is None: if self.vector_generation is not None and self.embedding_model is None:
raise ValueError("vector_generation requires embedding model and dimension compatibility") raise ValueError("vector_generation requires embedding model and dimension compatibility")
+10 -1
View File
@@ -125,6 +125,7 @@ class CorpusPipeline:
def __init__( def __init__(
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder, self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int, vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
embedding_id: str | None = None,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3, chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str | None = None, sparse_language: str = "italian", workspace_id: str | None = None, sparse_language: str = "italian",
candidate_evaluator: Callable[[CorpusManifest], object] | None = None, candidate_evaluator: Callable[[CorpusManifest], object] | None = None,
@@ -133,6 +134,7 @@ class CorpusPipeline:
self.sources = sources self.sources = sources
self.embedder = embedder self.embedder = embedder
self.vector_store = vector_store self.vector_store = vector_store
self.embedding_id = embedding_id or f"ollama/{embedding_model}"
self.embedding_model = embedding_model self.embedding_model = embedding_model
self.embedding_dimensions = embedding_dimensions self.embedding_dimensions = embedding_dimensions
self.chunk_policy = chunk_policy self.chunk_policy = chunk_policy
@@ -284,6 +286,7 @@ class CorpusPipeline:
source_by_id = {item.source_id: (source, item) for source, item in discovered} source_by_id = {item.source_id: (source, item) for source, item in discovered}
compatibility = _fingerprint({ compatibility = _fingerprint({
"pipeline": self.pipeline_version, "pipeline": self.pipeline_version,
"embedding_id": self.embedding_id,
"model": self.embedding_model, "model": self.embedding_model,
"dimensions": self.embedding_dimensions, "dimensions": self.embedding_dimensions,
"chunk_policy": asdict(self.chunk_policy), "chunk_policy": asdict(self.chunk_policy),
@@ -294,6 +297,7 @@ class CorpusPipeline:
"compatibility_fingerprint": compatibility, "compatibility_fingerprint": compatibility,
"pipeline_version": self.pipeline_version, "pipeline_version": self.pipeline_version,
"chunk_policy_version": self.chunk_policy.version, "chunk_policy_version": self.chunk_policy.version,
"embedding_id": self.embedding_id,
"embedding_model": self.embedding_model, "embedding_model": self.embedding_model,
"embedding_dimensions": self.embedding_dimensions, "embedding_dimensions": self.embedding_dimensions,
} }
@@ -492,7 +496,9 @@ class CorpusPipeline:
) for document in documents ) for document in documents
} }
manifest = CorpusManifest( manifest = CorpusManifest(
schema_version=2,
pipeline_version=self.pipeline_version, pipeline_version=self.pipeline_version,
embedding_id=self.embedding_id,
embedding_model=self.embedding_model, embedding_model=self.embedding_model,
embedding_dimensions=self.embedding_dimensions, embedding_dimensions=self.embedding_dimensions,
vector_generation=plan["generation"], vector_generation=plan["generation"],
@@ -723,7 +729,8 @@ class CorpusPipeline:
prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {} prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {}
fingerprints = {item.source_id: item.fingerprint for _, item in discovered} fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
compatibility = _fingerprint({ compatibility = _fingerprint({
"pipeline": self.pipeline_version, "model": self.embedding_model, "pipeline": self.pipeline_version, "embedding_id": self.embedding_id,
"model": self.embedding_model,
"dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy), "dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy),
}) })
previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None
@@ -761,7 +768,9 @@ class CorpusPipeline:
for document in documents for document in documents
} }
manifest = CorpusManifest( manifest = CorpusManifest(
schema_version=2,
pipeline_version=self.pipeline_version, pipeline_version=self.pipeline_version,
embedding_id=self.embedding_id,
embedding_model=self.embedding_model, embedding_model=self.embedding_model,
embedding_dimensions=self.embedding_dimensions, embedding_dimensions=self.embedding_dimensions,
vector_generation=generation, vector_generation=generation,
+2
View File
@@ -22,6 +22,7 @@ def build_preprocessing_pipeline(
vector_store: VectorStore, vector_store: VectorStore,
embedding_model: str, embedding_model: str,
embedding_dimensions: int, embedding_dimensions: int,
embedding_id: str | None = None,
chunk_policy: ChunkPolicy, chunk_policy: ChunkPolicy,
pipeline_version: str, pipeline_version: str,
retain_published_generations: int = 3, retain_published_generations: int = 3,
@@ -35,6 +36,7 @@ def build_preprocessing_pipeline(
sources=sources, sources=sources,
embedder=embedder, embedder=embedder,
vector_store=vector_store, vector_store=vector_store,
embedding_id=embedding_id,
embedding_model=embedding_model, embedding_model=embedding_model,
embedding_dimensions=embedding_dimensions, embedding_dimensions=embedding_dimensions,
chunk_policy=chunk_policy, chunk_policy=chunk_policy,
-2
View File
@@ -39,8 +39,6 @@ evidence:
embeddings: embeddings:
base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434 base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434
model: nomic-embed-text-v2-moe
dim: 768
batch_size: 32 batch_size: 32
# Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che # Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che
-2
View File
@@ -25,8 +25,6 @@ paths:
embeddings: embeddings:
base_url: ${THT_OLLAMA_URL} base_url: ${THT_OLLAMA_URL}
model: nomic-embed-text-v2-moe
dim: 768
batch_size: 64 batch_size: 64
# LOADING diretto del pgvector (server-only). Su workstation la lettura passa da # LOADING diretto del pgvector (server-only). Su workstation la lettura passa da
-2
View File
@@ -39,8 +39,6 @@ evidence:
embeddings: embeddings:
base_url: ${THT_OLLAMA_URL} # es. http://localhost:11434 base_url: ${THT_OLLAMA_URL} # es. http://localhost:11434
model: nomic-embed-text-v2-moe
dim: 768
batch_size: 32 batch_size: 32
vectors: vectors:
+7 -2
View File
@@ -528,8 +528,13 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
defer func() { _ = lock.Release() }() defer func() { _ = lock.Release() }()
} }
if args[0] == "restart" || args[0] == "update" || args[0] == "rollback" { if args[0] == "restart" || args[0] == "update" || args[0] == "rollback" {
if err := modelprojection.Generate(installation); err != nil { drift, err := modelprojection.Check(installation)
return commandUsageError(stderr, "installation model catalog could not be projected: "+err.Error()) if err != nil {
return commandUsageError(stderr, "installation model catalog could not be verified: "+err.Error())
}
if len(drift) > 0 {
return commandUsageError(stderr,
"installation model catalog changed; run tht start to apply the complete runtime projection")
} }
} }
controlled := compose.InstallationRunner{Installation: installation, Runner: runner} controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
+20
View File
@@ -1057,6 +1057,7 @@ func TestRunPiStatusUsesImageBundledPi(t *testing.T) {
func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T) { func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t) fixture.setEnvironment(t)
fixture.generateModelProjection(t)
if err := os.WriteFile(filepath.Join(fixture.projectDirectory, "docker", "core.Dockerfile"), []byte("ARG PI_VERSION=0.79.0\n"), 0o600); err != nil { if err := os.WriteFile(filepath.Join(fixture.projectDirectory, "docker", "core.Dockerfile"), []byte("ARG PI_VERSION=0.79.0\n"), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1079,6 +1080,7 @@ func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T
func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) { func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t) fixture.setEnvironment(t)
fixture.generateModelProjection(t)
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{"--installation", fixture.installationPath, "pi", "update", "--version", "0.81.0", "--source", "build"}, &stdout, &stderr) exitCode := run(context.Background(), []string{"--installation", fixture.installationPath, "pi", "update", "--version", "0.81.0", "--source", "build"}, &stdout, &stderr)
@@ -1095,6 +1097,7 @@ func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing
func TestRunPiRestartRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) { func TestRunPiRestartRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t) fixture.setEnvironment(t)
fixture.generateModelProjection(t)
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{ exitCode := run(context.Background(), []string{
@@ -1117,6 +1120,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
fixture.setEnvironment(t, secretPath) fixture.setEnvironment(t, secretPath)
fixture.generateModelProjection(t)
t.Setenv("THT_FAKE_PI_VERSION", "pi-restart-secret") t.Setenv("THT_FAKE_PI_VERSION", "pi-restart-secret")
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
@@ -1138,6 +1142,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) {
func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) { func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t) fixture.setEnvironment(t)
fixture.generateModelProjection(t)
var stdout, stderr bytes.Buffer var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{ exitCode := run(context.Background(), []string{
@@ -1152,6 +1157,21 @@ func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) {
} }
} }
func TestRunPiRestartRejectsCatalogDriftBeforeInvokingDocker(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t)
var stdout, stderr bytes.Buffer
exitCode := run(context.Background(), []string{
"--installation", fixture.installationPath, "pi", "restart", "--yes",
}, &stdout, &stderr)
if exitCode != 2 || !strings.Contains(stderr.String(), "run tht start") {
t.Fatalf("run() = %d, stderr=%q; want complete projection guidance", exitCode, stderr.String())
}
assertDockerNotInvoked(t, fixture)
}
func TestRunPiMaintenanceStatusAndRecoverConfirmationContract(t *testing.T) { func TestRunPiMaintenanceStatusAndRecoverConfirmationContract(t *testing.T) {
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n") fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
fixture.setEnvironment(t) fixture.setEnvironment(t)
@@ -78,6 +78,13 @@ func TestLoadRejectsInvalidModelCatalogDefaultsAndAuthentication(t *testing.T) {
catalog: strings.Replace(validModelCatalogYAML(), "id: ollama/qwen3-embedding:0.6b", "id: external/qwen3-embedding:0.6b", 1), catalog: strings.Replace(validModelCatalogYAML(), "id: ollama/qwen3-embedding:0.6b", "id: external/qwen3-embedding:0.6b", 1),
wantError: "ollama canonical id", wantError: "ollama canonical id",
}, },
{
name: "upstream model aliases must be unique within a provider",
catalog: strings.Replace(validModelCatalogYAML(),
" models:\n glm-5.3:\n",
" models:\n glm-alias:\n upstreamModel: glm-5.3\n session:\n contextWindow: 200000\n maxTokens: 131072\n glm-5.3:\n", 1),
wantError: "same upstreamModel",
},
{ {
name: "metadata default required", name: "metadata default required",
catalog: strings.Replace(validModelCatalogYAML(), " metadataGeneration: local/qwen\n", "", 1), catalog: strings.Replace(validModelCatalogYAML(), " metadataGeneration: local/qwen\n", "", 1),
@@ -14,6 +14,7 @@ const maxCatalogModels = 64
var catalogKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}$`) var catalogKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}$`)
var catalogModelIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`) var catalogModelIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`)
var catalogUpstreamModelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`)
var catalogAPIVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) var catalogAPIVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
// ModelCatalog is the only operator-authored source for model identity and runtime eligibility. // ModelCatalog is the only operator-authored source for model identity and runtime eligibility.
@@ -163,11 +164,23 @@ func (c ModelCatalog) Validate(environment map[string]string) error {
return err return err
} }
hasSession, providerHasMetadata := false, false hasSession, providerHasMetadata := false, false
upstreamModels := make(map[string]string)
for modelID, model := range provider.Models { for modelID, model := range provider.Models {
if !catalogKeyPattern.MatchString(modelID) { if !catalogKeyPattern.MatchString(modelID) {
return fmt.Errorf("modelCatalog model %q/%q is invalid", providerID, modelID) return fmt.Errorf("modelCatalog model %q/%q is invalid", providerID, modelID)
} }
canonical := providerID + "/" + modelID canonical := providerID + "/" + modelID
upstream := model.UpstreamModel
if upstream == "" {
upstream = modelID
}
if !catalogUpstreamModelPattern.MatchString(upstream) {
return fmt.Errorf("modelCatalog model %q upstreamModel is invalid", canonical)
}
if existing, duplicate := upstreamModels[upstream]; duplicate {
return fmt.Errorf("modelCatalog models %q and %q use the same upstreamModel", existing, canonical)
}
upstreamModels[upstream] = canonical
if model.Session == nil && model.MetadataGeneration == nil { if model.Session == nil && model.MetadataGeneration == nil {
return fmt.Errorf("modelCatalog model %q has no runtime use", canonical) return fmt.Errorf("modelCatalog model %q has no runtime use", canonical)
} }
@@ -119,9 +119,10 @@ func Run(request Request) error {
if !filepath.IsAbs(request.InstallationPath) || filepath.Base(request.InstallationPath) != "thothii-installation.yaml" { if !filepath.IsAbs(request.InstallationPath) || filepath.Base(request.InstallationPath) != "thothii-installation.yaml" {
return errors.New("installation migration requires an absolute thothii-installation.yaml path") return errors.New("installation migration requires an absolute thothii-installation.yaml path")
} }
if !filepath.IsAbs(request.OutputPath) || filepath.Base(request.OutputPath) != "thothii-installation.yaml" || outputExtension := strings.ToLower(filepath.Ext(request.OutputPath))
if !filepath.IsAbs(request.OutputPath) || outputExtension != ".yaml" && outputExtension != ".yml" ||
filepath.Clean(request.OutputPath) == filepath.Clean(request.InstallationPath) { filepath.Clean(request.OutputPath) == filepath.Clean(request.InstallationPath) {
return errors.New("installation migration output must be a different absolute thothii-installation.yaml path") return errors.New("installation migration output must be a different absolute YAML path")
} }
if request.SessionDefault == "" || request.EmbeddingID == "" || request.EmbeddingDimensions <= 0 { if request.SessionDefault == "" || request.EmbeddingID == "" || request.EmbeddingDimensions <= 0 {
return errors.New("installation migration requires session default, embedding id, and positive embedding dimensions") return errors.New("installation migration requires session default, embedding id, and positive embedding dimensions")
@@ -69,6 +69,19 @@ func TestRunReportsProviderAuthenticationConflictAndPublishesNothing(t *testing.
} }
} }
func TestRunAcceptsVersionedYAMLCandidateFilename(t *testing.T) {
fixture := migrationFixture(t, false)
request := fixture.request()
request.OutputPath = filepath.Join(filepath.Dir(request.OutputPath), "thothii-installation.v2.yaml")
if err := Run(request); err != nil {
t.Fatalf("Run() error = %v", err)
}
if info, err := os.Stat(request.OutputPath); err != nil || !info.Mode().IsRegular() {
t.Fatalf("versioned candidate: info=%v err=%v", info, err)
}
}
type migrationTestFixture struct { type migrationTestFixture struct {
legacy string legacy string
output string output string
@@ -21,6 +21,8 @@ const (
ComposeFile = "compose.models.yaml" ComposeFile = "compose.models.yaml"
) )
var renameProjectionDirectory = os.Rename
type runtimeCatalog struct { type runtimeCatalog struct {
SchemaVersion int `json:"schemaVersion"` SchemaVersion int `json:"schemaVersion"`
DefaultSession string `json:"defaultSession"` DefaultSession string `json:"defaultSession"`
@@ -87,10 +89,11 @@ func Render(installation config.Installation) (map[string][]byte, error) {
if model.Session == nil { if model.Session == nil {
continue continue
} }
enabled = append(enabled, model.ID)
if model.SessionAdapter == nil || model.SessionAdapter.Mode != "openai_compatible" { if model.SessionAdapter == nil || model.SessionAdapter.Mode != "openai_compatible" {
enabled = append(enabled, model.ID)
continue continue
} }
enabled = append(enabled, model.Provider+"/"+model.UpstreamModel)
provider := customProviders[model.Provider] provider := customProviders[model.Provider]
provider.BaseURL = model.Endpoint.BaseURL provider.BaseURL = model.Endpoint.BaseURL
provider.API = "openai-completions" provider.API = "openai-completions"
@@ -137,52 +140,84 @@ func Render(installation config.Installation) (map[string][]byte, error) {
}, nil }, nil
} }
// Generate atomically replaces each generated adapter after the complete candidate has rendered. // Generate publishes all adapters as one directory generation. A failed replacement restores the
// previous directory, so callers never observe a successfully returned mixed generation.
func Generate(installation config.Installation) error { func Generate(installation config.Installation) error {
artifacts, err := Render(installation) artifacts, err := Render(installation)
if err != nil { if err != nil {
return err return err
} }
paths := sortedArtifactPaths(artifacts) target := installation.GeneratedDirectory()
for _, relative := range paths { parent := filepath.Dir(target)
destination := filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(relative)) if err := os.MkdirAll(parent, 0o755); err != nil {
// The generated catalog contains references to secret environment variable names, never return fmt.Errorf("create model projection parent: %w", err)
// secret values. Core runs as an unprivileged container user and must be able to traverse }
// the bind-mounted host directories and read the projections. candidate, err := os.MkdirTemp(parent, ".model-projections-candidate-*")
if err != nil {
return fmt.Errorf("create model projection candidate: %w", err)
}
defer func() { _ = os.RemoveAll(candidate) }()
if err := writeProjectionCandidate(candidate, artifacts); err != nil {
return err
}
info, statErr := os.Lstat(target)
if os.IsNotExist(statErr) {
if err := renameProjectionDirectory(candidate, target); err != nil {
return fmt.Errorf("publish model projection generation: %w", err)
}
return nil
}
if statErr != nil {
return fmt.Errorf("inspect current model projection generation: %w", statErr)
}
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("current model projection path is not a regular directory")
}
previous, err := absentTemporaryPath(parent)
if err != nil {
return fmt.Errorf("reserve previous model projection generation: %w", err)
}
if err := renameProjectionDirectory(target, previous); err != nil {
return fmt.Errorf("prepare model projection generation replacement: %w", err)
}
if err := renameProjectionDirectory(candidate, target); err != nil {
if restoreErr := renameProjectionDirectory(previous, target); restoreErr != nil {
return fmt.Errorf("publish model projection generation: %v; restore previous generation: %w", err, restoreErr)
}
return fmt.Errorf("publish model projection generation: %w", err)
}
_ = os.RemoveAll(previous)
return nil
}
func writeProjectionCandidate(directory string, artifacts map[string][]byte) error {
if err := os.Chmod(directory, 0o755); err != nil {
return fmt.Errorf("protect model projection candidate: %w", err)
}
for _, relative := range sortedArtifactPaths(artifacts) {
destination := filepath.Join(directory, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
return fmt.Errorf("create model projection directory: %w", err) return fmt.Errorf("create model projection directory: %w", err)
} }
if err := os.Chmod(filepath.Dir(destination), 0o755); err != nil { if err := os.WriteFile(destination, artifacts[relative], 0o644); err != nil {
return fmt.Errorf("protect model projection directory: %w", err)
}
temporary, err := os.CreateTemp(filepath.Dir(destination), ".projection-*")
if err != nil {
return fmt.Errorf("create model projection candidate: %w", err)
}
temporaryName := temporary.Name()
published := false
defer func() {
if !published {
_ = os.Remove(temporaryName)
}
}()
if err := temporary.Chmod(0o644); err == nil {
_, err = temporary.Write(artifacts[relative])
}
if closeErr := temporary.Close(); err == nil {
err = closeErr
}
if err != nil {
return fmt.Errorf("write model projection candidate: %w", err) return fmt.Errorf("write model projection candidate: %w", err)
} }
if err := os.Rename(temporaryName, destination); err != nil {
return fmt.Errorf("publish model projection: %w", err)
}
published = true
} }
return nil return nil
} }
func absentTemporaryPath(parent string) (string, error) {
path, err := os.MkdirTemp(parent, ".model-projections-previous-*")
if err != nil {
return "", err
}
if err := os.Remove(path); err != nil {
return "", err
}
return path, nil
}
// Check returns relative artifact names whose current bytes differ from the catalog projection. // Check returns relative artifact names whose current bytes differ from the catalog projection.
func Check(installation config.Installation) ([]string, error) { func Check(installation config.Installation) ([]string, error) {
artifacts, err := Render(installation) artifacts, err := Render(installation)
@@ -2,6 +2,7 @@ package modelprojection
import ( import (
"bytes" "bytes"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -41,9 +42,12 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T)
t.Fatalf("Pi models projection must contain only custom providers:\n%s", piModels) t.Fatalf("Pi models projection must contain only custom providers:\n%s", piModels)
} }
settings := string(first[PiSettingsFile]) settings := string(first[PiSettingsFile])
if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen"`) { if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen2.5:7b"`) {
t.Fatalf("Pi settings projection missing enabled session models:\n%s", settings) t.Fatalf("Pi settings projection missing enabled session models:\n%s", settings)
} }
if !strings.Contains(piModels, `"id": "qwen2.5:7b"`) || !strings.Contains(catalog, `"id": "local/qwen"`) {
t.Fatalf("Pi projection must map canonical catalog keys to upstream model IDs")
}
compose := string(first[ComposeFile]) compose := string(first[ComposeFile])
for _, expected := range []string{"THT_MODEL_CATALOG_FILE", "THT_DEFAULT_SESSION_MODEL", "zai/glm-5.3", "THT_INTERNAL_EMBEDDING_ID", "THT_INTERNAL_EMBEDDING_MODEL", "OLLAMA_MODEL", installation.GeneratedModelCatalogPath()} { for _, expected := range []string{"THT_MODEL_CATALOG_FILE", "THT_DEFAULT_SESSION_MODEL", "zai/glm-5.3", "THT_INTERNAL_EMBEDDING_ID", "THT_INTERNAL_EMBEDDING_MODEL", "OLLAMA_MODEL", installation.GeneratedModelCatalogPath()} {
if !strings.Contains(compose, expected) { if !strings.Contains(compose, expected) {
@@ -52,6 +56,46 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T)
} }
} }
func TestGenerateRestoresWholePreviousGenerationWhenPublishFails(t *testing.T) {
installation := projectionFixture(t)
if err := Generate(installation); err != nil {
t.Fatalf("Generate() initial error = %v", err)
}
before := make(map[string][]byte)
for _, path := range []string{CatalogFile, PiModelsFile, PiSettingsFile, ComposeFile} {
contents, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
if err != nil {
t.Fatal(err)
}
before[path] = contents
}
provider := installation.ModelCatalog.Providers["zai"]
model := provider.Models["glm-5.3"]
model.Label = "Changed label"
provider.Models["glm-5.3"] = model
installation.ModelCatalog.Providers["zai"] = provider
originalRename := renameProjectionDirectory
t.Cleanup(func() { renameProjectionDirectory = originalRename })
renames := 0
renameProjectionDirectory = func(oldPath, newPath string) error {
renames++
if renames == 2 {
return errors.New("injected publish failure")
}
return os.Rename(oldPath, newPath)
}
if err := Generate(installation); err == nil {
t.Fatal("Generate() error = nil, want injected publish failure")
}
for path, expected := range before {
actual, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
if err != nil || !bytes.Equal(actual, expected) {
t.Fatalf("artifact %q changed after failed generation: err=%v", path, err)
}
}
}
func TestGeneratePublishesAllArtifactsAndCheckDetectsDrift(t *testing.T) { func TestGeneratePublishesAllArtifactsAndCheckDetectsDrift(t *testing.T) {
installation := projectionFixture(t) installation := projectionFixture(t)
if err := Generate(installation); err != nil { if err := Generate(installation); err != nil {
@@ -84,7 +128,7 @@ func projectionFixture(t *testing.T) config.Installation {
Providers: map[string]config.ModelProvider{ Providers: map[string]config.ModelProvider{
"deepseek": {Authentication: config.ModelAuthentication{Mode: "pi_auth"}, Session: &config.ModelSessionAdapter{Mode: "pi_builtin"}, Models: map[string]config.CatalogModel{"deepseek-v4-pro": {Session: &config.SessionModel{}}}}, "deepseek": {Authentication: config.ModelAuthentication{Mode: "pi_auth"}, Session: &config.ModelSessionAdapter{Mode: "pi_builtin"}, Models: map[string]config.CatalogModel{"deepseek-v4-pro": {Session: &config.SessionModel{}}}},
"zai": {Endpoint: &config.ModelEndpoint{BaseURL: "https://api.z.ai/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, Models: map[string]config.CatalogModel{"glm-5.3": {Label: "GLM-5.3", Session: &config.SessionModel{Reasoning: true, ContextWindow: 200000, MaxTokens: 131072}}}}, "zai": {Endpoint: &config.ModelEndpoint{BaseURL: "https://api.z.ai/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, Models: map[string]config.CatalogModel{"glm-5.3": {Label: "GLM-5.3", Session: &config.SessionModel{Reasoning: true, ContextWindow: 200000, MaxTokens: 131072}}}},
"local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}}, "local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {UpstreamModel: "qwen2.5:7b", Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}},
}, },
}, },
} }