fix: harden model catalog projections
This commit is contained in:
@@ -268,6 +268,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
|||||||
);
|
);
|
||||||
|
|
||||||
const listModels = deps?.listModels ?? createPiModelLister(config, {
|
const listModels = deps?.listModels ?? createPiModelLister(config, {
|
||||||
|
modelCatalog: runtimeModelCatalog,
|
||||||
warn: (detail) => app.log.warn(
|
warn: (detail) => app.log.warn(
|
||||||
{ component: "pi-model-list", detail },
|
{ component: "pi-model-list", detail },
|
||||||
"Pi enabled-model configuration warning",
|
"Pi enabled-model configuration warning",
|
||||||
|
|||||||
@@ -3,16 +3,17 @@ import type { CatalogDatabase } from "../repository.js";
|
|||||||
|
|
||||||
const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$";
|
const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$";
|
||||||
const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$";
|
const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$";
|
||||||
|
const historicalOrCanonicalModelPattern = `(${legacyModelPattern})|(${canonicalModelPattern})`;
|
||||||
|
|
||||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||||
await sql.raw(`alter table description_generation_runs
|
await sql.raw(`alter table description_generation_runs
|
||||||
drop constraint description_generation_runs_model_id_check,
|
drop constraint description_generation_runs_model_id_check,
|
||||||
add constraint description_generation_runs_model_id_check
|
add constraint description_generation_runs_model_id_check
|
||||||
check (model_id ~ '${canonicalModelPattern}')`).execute(db);
|
check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
|
||||||
await sql.raw(`alter table sensitive_data_suggestion_runs
|
await sql.raw(`alter table sensitive_data_suggestion_runs
|
||||||
drop constraint sensitive_data_suggestion_runs_model_id_check,
|
drop constraint sensitive_data_suggestion_runs_model_id_check,
|
||||||
add constraint sensitive_data_suggestion_runs_model_id_check
|
add constraint sensitive_data_suggestion_runs_model_id_check
|
||||||
check (model_id ~ '${canonicalModelPattern}')`).execute(db);
|
check (model_id ~ '${historicalOrCanonicalModelPattern}')`).execute(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import {
|
|||||||
readConfiguredPiAgentFile,
|
readConfiguredPiAgentFile,
|
||||||
validateDeclarativePiConfig,
|
validateDeclarativePiConfig,
|
||||||
} from "./managed-config.js";
|
} from "./managed-config.js";
|
||||||
|
import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
|
||||||
|
|
||||||
export interface PiModel {
|
export interface PiModel {
|
||||||
provider: string;
|
provider: string;
|
||||||
@@ -30,6 +31,7 @@ interface Opts {
|
|||||||
nowMs?: () => number;
|
nowMs?: () => number;
|
||||||
loadEnabledModels?: () => PiEnabledModelsResult;
|
loadEnabledModels?: () => PiEnabledModelsResult;
|
||||||
readModelsStore?: () => string | undefined;
|
readModelsStore?: () => string | undefined;
|
||||||
|
modelCatalog?: RuntimeModelCatalog;
|
||||||
warn?: (message: string) => void;
|
warn?: (message: string) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -82,9 +84,23 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModels
|
|||||||
const byCompositeId = new Map(
|
const byCompositeId = new Map(
|
||||||
available.map((model) => [`${model.provider}/${model.id}`, model]),
|
available.map((model) => [`${model.provider}/${model.id}`, model]),
|
||||||
);
|
);
|
||||||
|
const catalogByPiId = new Map(
|
||||||
|
(opts.modelCatalog?.sessionModels() ?? []).map((model) => [
|
||||||
|
`${model.provider}/${model.upstreamModel}`,
|
||||||
|
model,
|
||||||
|
]),
|
||||||
|
);
|
||||||
const models = enabled.ids.flatMap((id) => {
|
const models = enabled.ids.flatMap((id) => {
|
||||||
const model = byCompositeId.get(id);
|
const model = byCompositeId.get(id);
|
||||||
return model ? [model] : [];
|
if (!model) return [];
|
||||||
|
const catalogModel = catalogByPiId.get(id);
|
||||||
|
return [catalogModel ? {
|
||||||
|
...model,
|
||||||
|
provider: catalogModel.provider,
|
||||||
|
id: catalogModel.model,
|
||||||
|
name: catalogModel.label,
|
||||||
|
reasoning: catalogModel.session?.reasoning ?? model.reasoning,
|
||||||
|
} : model];
|
||||||
});
|
});
|
||||||
if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available");
|
if (models.length === 0) opts.warn?.("No Pi-enabled models are currently available");
|
||||||
cache = { at: now(), models };
|
cache = { at: now(), models };
|
||||||
|
|||||||
@@ -260,8 +260,11 @@ export class PiProcessManager {
|
|||||||
const thinking = o.thinking ?? this.cfg.defaults.thinking;
|
const thinking = o.thinking ?? this.cfg.defaults.thinking;
|
||||||
|
|
||||||
if (provider && model) {
|
if (provider && model) {
|
||||||
|
const upstreamModel = this.modelCatalog.sessionModels()
|
||||||
|
.find((entry) => entry.provider === provider && entry.model === model)
|
||||||
|
?.upstreamModel ?? model;
|
||||||
const response = await rt.rpc.request(
|
const response = await rt.rpc.request(
|
||||||
{ type: "set_model", provider, modelId: model } as object & { type: string },
|
{ type: "set_model", provider, modelId: upstreamModel } as object & { type: string },
|
||||||
);
|
);
|
||||||
rt.bridge.setContextWindow(response?.data?.contextWindow);
|
rt.bridge.setContextWindow(response?.data?.contextWindow);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ export function createPiProviderSmoke(
|
|||||||
|| catalog.defaultSession !== null;
|
|| catalog.defaultSession !== null;
|
||||||
const catalogModel = catalog.sessionModels()
|
const catalogModel = catalog.sessionModels()
|
||||||
.find((entry) => entry.provider === canonicalProvider && entry.model === model);
|
.find((entry) => entry.provider === canonicalProvider && entry.model === model);
|
||||||
|
const upstreamModel = catalogModel?.upstreamModel ?? model;
|
||||||
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
const credentialName = catalogModel?.authentication.mode === "secret_env"
|
||||||
? catalogModel.authentication.apiKeyEnv
|
? catalogModel.authentication.apiKeyEnv
|
||||||
: undefined;
|
: undefined;
|
||||||
@@ -130,7 +131,7 @@ export function createPiProviderSmoke(
|
|||||||
const capabilityGuard = failOnUnexpectedCapabilities(rpc);
|
const capabilityGuard = failOnUnexpectedCapabilities(rpc);
|
||||||
const turn = async (): Promise<void> => {
|
const turn = async (): Promise<void> => {
|
||||||
requireSuccessfulResponse(await rpc.request({
|
requireSuccessfulResponse(await rpc.request({
|
||||||
type: "set_model", provider: canonicalProvider, modelId: model,
|
type: "set_model", provider: canonicalProvider, modelId: upstreamModel,
|
||||||
} as object & { type: string }));
|
} as object & { type: string }));
|
||||||
requireSuccessfulResponse(await rpc.request({
|
requireSuccessfulResponse(await rpc.request({
|
||||||
type: "set_thinking_level", level: reasoning,
|
type: "set_thinking_level", level: reasoning,
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbed
|
|||||||
}
|
}
|
||||||
const model = requireString(embeddings, "model");
|
const model = requireString(embeddings, "model");
|
||||||
return {
|
return {
|
||||||
id: `ollama/${model}`,
|
id: optionalString(embeddings, "id") ?? `ollama/${model}`,
|
||||||
model,
|
model,
|
||||||
dimensions: requireNumber(embeddings, "dimensions"),
|
dimensions: requireNumber(embeddings, "dimensions"),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -274,6 +274,8 @@ export function renderRuntimeConfig(
|
|||||||
},
|
},
|
||||||
embedding: {
|
embedding: {
|
||||||
provider: "ollama_internal",
|
provider: "ollama_internal",
|
||||||
|
id: semanticRuntime.internalEmbeddingId
|
||||||
|
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
|
||||||
model: semanticRuntime.internalEmbeddingModel,
|
model: semanticRuntime.internalEmbeddingModel,
|
||||||
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
||||||
},
|
},
|
||||||
@@ -287,6 +289,8 @@ export function renderRuntimeConfig(
|
|||||||
embeddings: {
|
embeddings: {
|
||||||
provider: "ollama_internal",
|
provider: "ollama_internal",
|
||||||
base_url: semanticRuntime.internalEmbeddingUrl,
|
base_url: semanticRuntime.internalEmbeddingUrl,
|
||||||
|
id: semanticRuntime.internalEmbeddingId
|
||||||
|
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
|
||||||
model: semanticRuntime.internalEmbeddingModel,
|
model: semanticRuntime.internalEmbeddingModel,
|
||||||
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { spawnSync } from "node:child_process";
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
||||||
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
|
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
|
||||||
import { Pool } from "pg";
|
import { Pool } from "pg";
|
||||||
@@ -33,7 +34,30 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
|||||||
await upDescriptionGeneration(db);
|
await upDescriptionGeneration(db);
|
||||||
await upSensitiveSuggestionRuns(db);
|
await upSensitiveSuggestionRuns(db);
|
||||||
await upAiTokenUsage(db);
|
await upAiTokenUsage(db);
|
||||||
|
const historicalDatabaseId = randomUUID();
|
||||||
|
await db.insertInto("workspaceDatabases").values({
|
||||||
|
id: historicalDatabaseId,
|
||||||
|
workspaceId: "migration-history",
|
||||||
|
engine: "postgres",
|
||||||
|
databaseName: "warehouse",
|
||||||
|
schemaName: "public",
|
||||||
|
}).execute();
|
||||||
|
await db.insertInto("descriptionGenerationRuns").values({
|
||||||
|
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
|
||||||
|
modelId: "openai-mini", language: "en", status: "completed", total: 1,
|
||||||
|
processed: 1, generated: 1,
|
||||||
|
}).execute();
|
||||||
|
await db.insertInto("sensitiveDataSuggestionRuns").values({
|
||||||
|
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
|
||||||
|
modelId: "openai-mini", status: "completed", total: 1,
|
||||||
|
suggestedSensitive: 1,
|
||||||
|
}).execute();
|
||||||
await upCanonicalModelIds(db);
|
await upCanonicalModelIds(db);
|
||||||
|
await expect(db.insertInto("descriptionGenerationRuns").values({
|
||||||
|
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
|
||||||
|
modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1,
|
||||||
|
processed: 1, generated: 1,
|
||||||
|
}).execute()).resolves.toBeDefined();
|
||||||
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
||||||
await upRuntimeSequencePrivileges(db);
|
await upRuntimeSequencePrivileges(db);
|
||||||
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import { join } from "node:path";
|
|||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { createPiModelLister } from "../src/pi/list-models.js";
|
import { createPiModelLister } from "../src/pi/list-models.js";
|
||||||
import { loadConfig } from "../src/config.js";
|
import { loadConfig } from "../src/config.js";
|
||||||
|
import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";
|
||||||
|
|
||||||
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
|
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
|
||||||
|
|
||||||
@@ -44,6 +45,36 @@ test("createPiModelLister returns mapped PiModel[] from get_available_models", a
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("catalog listing translates upstream Pi IDs back to canonical model keys", async () => {
|
||||||
|
const script = scriptWith([
|
||||||
|
{ provider: "local", id: "qwen2.5:7b", name: "Upstream label", reasoning: false },
|
||||||
|
]);
|
||||||
|
const model: RuntimeModel = {
|
||||||
|
id: "local/qwen", provider: "local", model: "qwen", label: "Catalog Qwen",
|
||||||
|
upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
|
||||||
|
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
|
||||||
|
session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 },
|
||||||
|
};
|
||||||
|
const modelCatalog: RuntimeModelCatalog = {
|
||||||
|
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
|
||||||
|
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||||
|
...noManagedModels,
|
||||||
|
modelCatalog,
|
||||||
|
loadEnabledModels: enabled("local/qwen2.5:7b"),
|
||||||
|
spawnFn: () => spawn("node", [FAKE, script]) as any,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(lister()).resolves.toEqual([{
|
||||||
|
provider: "local", id: "qwen", name: "Catalog Qwen", reasoning: true,
|
||||||
|
}]);
|
||||||
|
} finally {
|
||||||
|
rmSync(path.dirname(script), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
|
test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
|
||||||
const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
|
const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -718,6 +718,54 @@ test.each([["OpenAI", "openai"], ["gemini", "google"]])(
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
test("set_model translates a canonical catalog key to its upstream Pi model ID", async () => {
|
||||||
|
const root = mkdtempSync(path.join(tmpdir(), "thothii-upstream-model-"));
|
||||||
|
const agentDir = path.join(root, "agent");
|
||||||
|
mkdirSync(agentDir, { mode: 0o700 });
|
||||||
|
writeFileSync(path.join(agentDir, "models.json"), JSON.stringify({
|
||||||
|
providers: {
|
||||||
|
local: {
|
||||||
|
baseUrl: "http://ollama:11434/v1", apiKey: "local",
|
||||||
|
models: [{ id: "qwen2.5:7b" }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}), { mode: 0o600 });
|
||||||
|
vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
|
||||||
|
const child = recordingChild();
|
||||||
|
child.stderr.resume = () => {};
|
||||||
|
child.stdin.write = (data: unknown) => {
|
||||||
|
const request = JSON.parse(String(data));
|
||||||
|
child._writes.push(String(data));
|
||||||
|
if (request.id) {
|
||||||
|
queueMicrotask(() => child.stdout.emit("data", `${JSON.stringify({
|
||||||
|
type: "response", id: request.id, success: true,
|
||||||
|
})}\n`));
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
const model: RuntimeModel = {
|
||||||
|
id: "local/qwen", provider: "local", model: "qwen", label: "Qwen",
|
||||||
|
upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
|
||||||
|
authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
|
||||||
|
session: { reasoning: false, contextWindow: 32768, maxTokens: 8192 },
|
||||||
|
};
|
||||||
|
const modelCatalog: RuntimeModelCatalog = {
|
||||||
|
defaultSession: model.id, defaultMetadataGeneration: null, embedding: null,
|
||||||
|
sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
|
||||||
|
};
|
||||||
|
const mgr = new PiProcessManager(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
|
||||||
|
modelCatalog, authProviders: () => new Set(), spawnFn: () => child as any,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await mgr.spawnFor("upstream-model", { provider: "local", model: "qwen" });
|
||||||
|
expect(child._writes.join("")).toContain('"modelId":"qwen2.5:7b"');
|
||||||
|
} finally {
|
||||||
|
mgr.teardown("upstream-model");
|
||||||
|
vi.unstubAllEnvs();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test.each(["installation-local", "private-compatible"])(
|
test.each(["installation-local", "private-compatible"])(
|
||||||
"provider %s configured with a literal apiKey spawns without a managed key",
|
"provider %s configured with a literal apiKey spawns without a managed key",
|
||||||
async (provider) => {
|
async (provider) => {
|
||||||
|
|||||||
@@ -300,11 +300,22 @@ test("provider smoke makes one configured request from an isolated no-capability
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
const smokeModel: RuntimeModel = {
|
||||||
|
id: "zai/catalog-glm", provider: "zai", model: "catalog-glm", label: "GLM",
|
||||||
|
upstreamModel: "glm-5.2", authentication: { mode: "pi_auth" },
|
||||||
|
sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true },
|
||||||
|
};
|
||||||
|
const smokeCatalog: RuntimeModelCatalog = {
|
||||||
|
defaultSession: smokeModel.id, defaultMetadataGeneration: null, embedding: null,
|
||||||
|
sessionModels: () => [smokeModel], metadataModels: () => [],
|
||||||
|
hasSession: (id) => id === smokeModel.id,
|
||||||
|
};
|
||||||
const smoke = createPiProviderSmoke(loadConfig({
|
const smoke = createPiProviderSmoke(loadConfig({
|
||||||
THT_HARNESS_DIR: "/app/harness",
|
THT_HARNESS_DIR: "/app/harness",
|
||||||
PI_BIN: "/usr/local/bin/pi",
|
PI_BIN: "/usr/local/bin/pi",
|
||||||
THT_DATA_ROOT: "/mounted-workflow-state",
|
THT_DATA_ROOT: "/mounted-workflow-state",
|
||||||
}), {
|
}), {
|
||||||
|
modelCatalog: smokeCatalog,
|
||||||
spawnFn: (...args) => {
|
spawnFn: (...args) => {
|
||||||
spawns.push(args);
|
spawns.push(args);
|
||||||
expect(args[2].cwd).not.toBe("/app/harness");
|
expect(args[2].cwd).not.toBe("/app/harness");
|
||||||
@@ -324,7 +335,7 @@ test("provider smoke makes one configured request from an isolated no-capability
|
|||||||
});
|
});
|
||||||
|
|
||||||
await expect(smoke({
|
await expect(smoke({
|
||||||
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750,
|
provider: "zai", model: "catalog-glm", reasoning: "medium", timeoutMs: 750,
|
||||||
})).resolves.toBeUndefined();
|
})).resolves.toBeUndefined();
|
||||||
expect(spawns).toHaveLength(1);
|
expect(spawns).toHaveLength(1);
|
||||||
expect(spawns[0][0]).toBe("/usr/local/bin/pi");
|
expect(spawns[0][0]).toBe("/usr/local/bin/pi");
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ test("derives the internal Qdrant and Ollama runtime shape from workspace v4 plu
|
|||||||
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
|
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
|
||||||
embeddings: {
|
embeddings: {
|
||||||
provider: "ollama_internal", base_url: "http://embedding:11434",
|
provider: "ollama_internal", base_url: "http://embedding:11434",
|
||||||
|
id: "ollama/qwen3-embedding:0.6b",
|
||||||
model: "qwen3-embedding:0.6b", dimensions: 1024,
|
model: "qwen3-embedding:0.6b", dimensions: 1024,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -35,7 +35,6 @@ resources:
|
|||||||
base_url: http://qdrant:6333
|
base_url: http://qdrant:6333
|
||||||
collection: server-sessions
|
collection: server-sessions
|
||||||
embeddings:
|
embeddings:
|
||||||
|
# id, model, and dimensions are injected from the Installation Model Catalog.
|
||||||
provider: ollama_internal
|
provider: ollama_internal
|
||||||
base_url: http://embedding:11434
|
base_url: http://embedding:11434
|
||||||
model: qwen3-embedding:0.6b
|
|
||||||
dimensions: 1024
|
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ generated/
|
|||||||
├── pi/
|
├── pi/
|
||||||
│ ├── models.json
|
│ ├── models.json
|
||||||
│ └── settings.json
|
│ └── settings.json
|
||||||
└── compose.model-catalog.yaml
|
└── compose.models.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
The normalized catalog is consumed by the backend. The Pi files and Compose override are boundary
|
The normalized catalog is consumed by the backend. The Pi files and Compose override are boundary
|
||||||
@@ -117,17 +117,19 @@ tht --installation "$INSTALLATION" start
|
|||||||
tht --installation "$INSTALLATION" doctor
|
tht --installation "$INSTALLATION" doctor
|
||||||
```
|
```
|
||||||
|
|
||||||
After editing `modelCatalog` or provider credentials, reload the current Pi image. Restart validates
|
After editing `modelCatalog` or provider credentials, apply the complete runtime projection with
|
||||||
the YAML and regenerates projections before recreating `core`:
|
the normal installation lifecycle, then run the Pi checks:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
tht --installation "$INSTALLATION" pi restart --yes --drain
|
tht --installation "$INSTALLATION" start
|
||||||
tht --installation "$INSTALLATION" pi doctor
|
tht --installation "$INSTALLATION" pi doctor
|
||||||
tht --installation "$INSTALLATION" pi test
|
tht --installation "$INSTALLATION" pi test
|
||||||
```
|
```
|
||||||
|
|
||||||
`tht pi update` changes the Pi version; it is not the configuration command. There is no
|
`tht pi restart`, `tht pi update`, and `tht pi rollback` refuse to run while generated model
|
||||||
`tht pi configure` and no separate apply command.
|
projections differ from `modelCatalog`: those commands recreate only `core`, so they must never
|
||||||
|
partially apply an embedding change. `tht pi update` changes the Pi version; it is not the
|
||||||
|
configuration command. There is no `tht pi configure` and no separate apply command.
|
||||||
|
|
||||||
## Migrating a legacy installation
|
## Migrating a legacy installation
|
||||||
|
|
||||||
@@ -136,7 +138,7 @@ files, but never modifies them. Supply the facts that cannot be inferred safely
|
|||||||
candidate:
|
candidate:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
tht --installation /absolute/path/legacy-installation.yaml installation migrate \
|
tht --installation /absolute/path/legacy/thothii-installation.yaml installation migrate \
|
||||||
--output /absolute/path/thothii-installation.v2.yaml \
|
--output /absolute/path/thothii-installation.v2.yaml \
|
||||||
--session-default zai/glm-5.3 \
|
--session-default zai/glm-5.3 \
|
||||||
--embedding-id ollama/qwen3-embedding:0.6b \
|
--embedding-id ollama/qwen3-embedding:0.6b \
|
||||||
|
|||||||
@@ -166,6 +166,8 @@ Generation is deterministic and published only after every candidate artifact va
|
|||||||
generation aborts start before Compose is invoked. `tht doctor` recomputes expected bytes and reports
|
generation aborts start before Compose is invoked. `tht doctor` recomputes expected bytes and reports
|
||||||
differences; no digest manifest or separate apply command exists. When projection bytes change,
|
differences; no digest manifest or separate apply command exists. When projection bytes change,
|
||||||
`tht start` recreates the affected services so they cannot continue with an older bind mount.
|
`tht start` recreates the affected services so they cannot continue with an older bind mount.
|
||||||
|
Pi-only restart, update, and rollback operations reject projection drift and direct the operator to
|
||||||
|
`tht start`, because applying only the core-facing files could leave embedding services stale.
|
||||||
|
|
||||||
Generated projections are not backed up. Restore validates the canonical installation descriptor,
|
Generated projections are not backed up. Restore validates the canonical installation descriptor,
|
||||||
regenerates every projection, and only then starts services. Base Compose files and `operator.env`
|
regenerates every projection, and only then starts services. Base Compose files and `operator.env`
|
||||||
|
|||||||
@@ -214,7 +214,7 @@ embeddings: {provider: ollama_internal, base_url: http://embedding:11434, model:
|
|||||||
assert cfg.vectors.writer.api_key == "writer"
|
assert cfg.vectors.writer.api_key == "writer"
|
||||||
|
|
||||||
|
|
||||||
def test_accepts_only_internal_ollama_embedding_contract(tmp_path):
|
def test_accepts_catalog_selected_internal_ollama_embedding_contract(tmp_path):
|
||||||
workspace = tmp_path / "workspace.yaml"
|
workspace = tmp_path / "workspace.yaml"
|
||||||
workspace.write_text(
|
workspace.write_text(
|
||||||
"""
|
"""
|
||||||
@@ -225,8 +225,9 @@ resources:
|
|||||||
embeddings:
|
embeddings:
|
||||||
provider: ollama_internal
|
provider: ollama_internal
|
||||||
base_url: http://embedding:11434
|
base_url: http://embedding:11434
|
||||||
model: qwen3-embedding:0.6b
|
id: ollama/bge-m3
|
||||||
dimensions: 1024
|
model: bge-m3
|
||||||
|
dimensions: 1536
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -234,8 +235,59 @@ resources:
|
|||||||
|
|
||||||
assert cfg.embeddings.provider == "ollama_internal"
|
assert cfg.embeddings.provider == "ollama_internal"
|
||||||
assert cfg.embeddings.base_url == "http://embedding:11434"
|
assert cfg.embeddings.base_url == "http://embedding:11434"
|
||||||
assert cfg.embeddings.model == "qwen3-embedding:0.6b"
|
assert cfg.embeddings.id == "ollama/bge-m3"
|
||||||
assert cfg.embeddings.dim == 1024
|
assert cfg.embeddings.model == "bge-m3"
|
||||||
|
assert cfg.embeddings.dim == 1536
|
||||||
|
|
||||||
|
|
||||||
|
def test_installation_embedding_projection_completes_model_free_runtime_source(
|
||||||
|
tmp_path, monkeypatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3")
|
||||||
|
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3")
|
||||||
|
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536")
|
||||||
|
workspace = tmp_path / "workspace.yaml"
|
||||||
|
workspace.write_text(
|
||||||
|
"""
|
||||||
|
dwh:
|
||||||
|
type: postgres_direct
|
||||||
|
connection: {database: analytics, schema: mart, user: reader, password: secret}
|
||||||
|
resources:
|
||||||
|
embeddings:
|
||||||
|
provider: ollama_internal
|
||||||
|
base_url: http://embedding:11434
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg = load_config(workspace)
|
||||||
|
|
||||||
|
assert cfg.embeddings.id == "ollama/bge-m3"
|
||||||
|
assert cfg.embeddings.model == "bge-m3"
|
||||||
|
assert cfg.embeddings.dim == 1536
|
||||||
|
|
||||||
|
|
||||||
|
def test_workspace_embedding_values_cannot_override_installation_projection(
|
||||||
|
tmp_path, monkeypatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_ID", "ollama/bge-m3")
|
||||||
|
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_MODEL", "bge-m3")
|
||||||
|
monkeypatch.setenv("THT_INTERNAL_EMBEDDING_DIMENSIONS", "1536")
|
||||||
|
workspace = tmp_path / "workspace.yaml"
|
||||||
|
workspace.write_text(
|
||||||
|
"""
|
||||||
|
dwh:
|
||||||
|
type: postgres_direct
|
||||||
|
connection: {database: analytics, schema: mart, user: reader, password: secret}
|
||||||
|
resources:
|
||||||
|
embeddings:
|
||||||
|
provider: ollama_internal
|
||||||
|
base_url: http://embedding:11434
|
||||||
|
model: workspace-owned-model
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ConfigError, match="proprietà dell'installazione|diverge"):
|
||||||
|
load_config(workspace)
|
||||||
|
|
||||||
|
|
||||||
def test_accepts_internal_qdrant_resource_contract(tmp_path):
|
def test_accepts_internal_qdrant_resource_contract(tmp_path):
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ def test_manifest_contains_provenance_without_credentials():
|
|||||||
manifest_id="manifest:abc",
|
manifest_id="manifest:abc",
|
||||||
created_at=datetime(2026, 7, 12, tzinfo=UTC),
|
created_at=datetime(2026, 7, 12, tzinfo=UTC),
|
||||||
pipeline_version="evidence-v1",
|
pipeline_version="evidence-v1",
|
||||||
|
embedding_id="ollama/nomic-embed-text",
|
||||||
embedding_model="nomic-embed-text",
|
embedding_model="nomic-embed-text",
|
||||||
embedding_dimensions=768,
|
embedding_dimensions=768,
|
||||||
documents=[document()],
|
documents=[document()],
|
||||||
@@ -52,6 +53,7 @@ def test_manifest_contains_provenance_without_credentials():
|
|||||||
assert "etag:abc" in payload
|
assert "etag:abc" in payload
|
||||||
assert "evidence-v1" in payload
|
assert "evidence-v1" in payload
|
||||||
assert "nomic-embed-text" in payload
|
assert "nomic-embed-text" in payload
|
||||||
|
assert '"embedding_id":"ollama/nomic-embed-text"' in payload
|
||||||
assert "api_key" not in payload
|
assert "api_key" not in payload
|
||||||
|
|
||||||
|
|
||||||
@@ -93,6 +95,25 @@ def test_manifest_validates_embedding_compatibility_fields():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_v1_manifest_without_canonical_embedding_id_remains_readable():
|
||||||
|
manifest = CorpusManifest.model_validate({
|
||||||
|
"schema_version": 1,
|
||||||
|
"embedding_model": "legacy-model",
|
||||||
|
"embedding_dimensions": 768,
|
||||||
|
})
|
||||||
|
|
||||||
|
assert manifest.embedding_id is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_v2_embedding_generation_requires_canonical_id():
|
||||||
|
with pytest.raises(ValidationError, match="embedding_id"):
|
||||||
|
CorpusManifest(
|
||||||
|
schema_version=2,
|
||||||
|
embedding_model="model-v2",
|
||||||
|
embedding_dimensions=768,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_canonical_metadata_rejects_secrets_and_non_json_values():
|
def test_canonical_metadata_rejects_secrets_and_non_json_values():
|
||||||
with pytest.raises(ValidationError, match="credential-like"):
|
with pytest.raises(ValidationError, match="credential-like"):
|
||||||
CanonicalDocument.model_validate(
|
CanonicalDocument.model_validate(
|
||||||
|
|||||||
@@ -105,11 +105,13 @@ def item(name, fingerprint):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None,
|
def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a",
|
||||||
|
embedding_id=None, policy=None,
|
||||||
retain=3, candidate_evaluator=None):
|
retain=3, candidate_evaluator=None):
|
||||||
return CorpusPipeline(
|
return CorpusPipeline(
|
||||||
store=CorpusStore(tmp_path / "corpus"), sources=[source],
|
store=CorpusStore(tmp_path / "corpus"), sources=[source],
|
||||||
embedder=embedder or Embedder(), vector_store=vectors or Vectors(),
|
embedder=embedder or Embedder(), vector_store=vectors or Vectors(),
|
||||||
|
embedding_id=embedding_id,
|
||||||
embedding_model=model, embedding_dimensions=3,
|
embedding_model=model, embedding_dimensions=3,
|
||||||
chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100),
|
chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100),
|
||||||
pipeline_version="evidence-v1",
|
pipeline_version="evidence-v1",
|
||||||
@@ -875,6 +877,24 @@ def test_model_or_chunk_policy_change_forces_full_rebuild(tmp_path):
|
|||||||
assert source.acquire_calls == ["fs:one"]
|
assert source.acquire_calls == ["fs:one"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_canonical_embedding_id_change_forces_full_rebuild_and_is_persisted(tmp_path):
|
||||||
|
one = item("one", "a")
|
||||||
|
first = pipeline(
|
||||||
|
tmp_path, Source([(one, "hello")]), model="same-upstream",
|
||||||
|
embedding_id="ollama/catalog-a",
|
||||||
|
).run()
|
||||||
|
assert first.manifest.embedding_id == "ollama/catalog-a"
|
||||||
|
|
||||||
|
source = Source([(one, "hello")])
|
||||||
|
changed = pipeline(
|
||||||
|
tmp_path, source, model="same-upstream", embedding_id="ollama/catalog-b",
|
||||||
|
).run()
|
||||||
|
|
||||||
|
assert changed.changed == ("fs:one",)
|
||||||
|
assert changed.manifest.embedding_id == "ollama/catalog-b"
|
||||||
|
assert source.acquire_calls == ["fs:one"]
|
||||||
|
|
||||||
|
|
||||||
def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path):
|
def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path):
|
||||||
one = item("one", "a")
|
one = item("one", "a")
|
||||||
good = pipeline(tmp_path, Source([(one, "old")]))
|
good = pipeline(tmp_path, Source([(one, "old")]))
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ def _write_cfg(tmp_path, raw):
|
|||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid", collection="psd", model="qwen3-embedding:0.6b"):
|
def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid",
|
||||||
|
collection="psd", model="qwen3-embedding:0.6b", dimensions=1024):
|
||||||
return {
|
return {
|
||||||
"schemaVersion": 1,
|
"schemaVersion": 1,
|
||||||
"workspace": {"schema_version": 3, "id": "psd", "name": "PSD", "language": "it"},
|
"workspace": {"schema_version": 3, "id": "psd", "name": "PSD", "language": "it"},
|
||||||
@@ -34,7 +35,10 @@ def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.inval
|
|||||||
"connection": {"host": "h", "port": 5432, "database": "warehouse", "schema": "dw", "user": "reader", "password": "secret"},
|
"connection": {"host": "h", "port": 5432, "database": "warehouse", "schema": "dw", "user": "reader", "password": "secret"},
|
||||||
},
|
},
|
||||||
"vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": collection, "collection_lifecycle": "self_heal"},
|
"vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": collection, "collection_lifecycle": "self_heal"},
|
||||||
"embeddings": {"provider": "ollama_internal", "base_url": "http://embedding:11434", "model": model, "dimensions": 1024},
|
"embeddings": {
|
||||||
|
"provider": "ollama_internal", "base_url": "http://embedding:11434",
|
||||||
|
"id": f"ollama/{model}", "model": model, "dimensions": dimensions,
|
||||||
|
},
|
||||||
"roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")},
|
"roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")},
|
||||||
"paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")},
|
"paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")},
|
||||||
}
|
}
|
||||||
@@ -52,6 +56,16 @@ def test_canonical_json_is_deterministic_and_key_ordered(cfg):
|
|||||||
assert keys == ["schemaVersion", "dwh", "vector", "embedding", "roots"]
|
assert keys == ["schemaVersion", "dwh", "vector", "embedding", "roots"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_catalog_embedding_identity_and_dimensions_drive_effective_config(tmp_path):
|
||||||
|
cfg = _write_cfg(tmp_path, _cfg(tmp_path, model="bge-m3", dimensions=1536))
|
||||||
|
document = __import__("json").loads(canonical_effective_config_json(cfg))
|
||||||
|
|
||||||
|
assert document["embedding"] == {
|
||||||
|
"id": "ollama/bge-m3", "model": "bge-m3", "dimensions": 1536,
|
||||||
|
}
|
||||||
|
assert document["vector"]["dimensions"] == 1536
|
||||||
|
|
||||||
|
|
||||||
def test_canonical_excludes_credentials_and_evidence(cfg):
|
def test_canonical_excludes_credentials_and_evidence(cfg):
|
||||||
doc = canonical_effective_config_json(cfg)
|
doc = canonical_effective_config_json(cfg)
|
||||||
assert "secret" not in doc
|
assert "secret" not in doc
|
||||||
|
|||||||
@@ -149,6 +149,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk
|
|||||||
"sources": [object()],
|
"sources": [object()],
|
||||||
"embedder": object(),
|
"embedder": object(),
|
||||||
"vector_store": object(),
|
"vector_store": object(),
|
||||||
|
"embedding_id": "ollama/model",
|
||||||
"embedding_model": "model",
|
"embedding_model": "model",
|
||||||
"embedding_dimensions": 3,
|
"embedding_dimensions": 3,
|
||||||
"chunk_policy": object(),
|
"chunk_policy": object(),
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ from tht.config import EmbeddingsConfig
|
|||||||
|
|
||||||
|
|
||||||
def _cfg(**kw):
|
def _cfg(**kw):
|
||||||
|
kw.setdefault("model", "nomic-embed-text-v2-moe")
|
||||||
|
kw.setdefault("dim", 768)
|
||||||
emb = EmbeddingsConfig(base_url="http://localhost:11434", **kw)
|
emb = EmbeddingsConfig(base_url="http://localhost:11434", **kw)
|
||||||
return SimpleNamespace(embeddings=emb)
|
return SimpleNamespace(embeddings=emb)
|
||||||
|
|
||||||
|
|||||||
@@ -284,6 +284,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
|
|||||||
sources,
|
sources,
|
||||||
embedder,
|
embedder,
|
||||||
vector_store,
|
vector_store,
|
||||||
|
embedding_id,
|
||||||
embedding_model,
|
embedding_model,
|
||||||
embedding_dimensions,
|
embedding_dimensions,
|
||||||
chunk_policy,
|
chunk_policy,
|
||||||
@@ -293,6 +294,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
|
|||||||
candidate_evaluator,
|
candidate_evaluator,
|
||||||
):
|
):
|
||||||
calls["init"] = {
|
calls["init"] = {
|
||||||
|
"embedding_id": embedding_id,
|
||||||
"embedding_model": embedding_model,
|
"embedding_model": embedding_model,
|
||||||
"embedding_dimensions": embedding_dimensions,
|
"embedding_dimensions": embedding_dimensions,
|
||||||
"pipeline_version": pipeline_version,
|
"pipeline_version": pipeline_version,
|
||||||
@@ -314,6 +316,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
|
|||||||
|
|
||||||
command.run_from_config(config)
|
command.run_from_config(config)
|
||||||
|
|
||||||
|
assert calls["init"]["embedding_id"] == "ollama/qwen3-embedding:0.6b"
|
||||||
assert calls["init"]["sparse_language"] == "english"
|
assert calls["init"]["sparse_language"] == "english"
|
||||||
assert calls["init"]["candidate_evaluator"] is None
|
assert calls["init"]["candidate_evaluator"] is None
|
||||||
assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
|
assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
|
||||||
|
|||||||
@@ -187,6 +187,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
|||||||
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
|
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
|
||||||
embedder=embedder,
|
embedder=embedder,
|
||||||
vector_store=vector_store,
|
vector_store=vector_store,
|
||||||
|
embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}",
|
||||||
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
|
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
|
||||||
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
|
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
|
||||||
pipeline_version="evidence-v1",
|
pipeline_version="evidence-v1",
|
||||||
@@ -222,6 +223,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
|
|||||||
pipeline = build_preprocessing_pipeline(
|
pipeline = build_preprocessing_pipeline(
|
||||||
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
|
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
|
||||||
embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True),
|
embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True),
|
||||||
|
embedding_id=cfg.embeddings.id or f"ollama/{cfg.embeddings.model}",
|
||||||
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
|
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
|
||||||
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
|
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
|
||||||
pipeline_version="evidence-v1",
|
pipeline_version="evidence-v1",
|
||||||
|
|||||||
+84
-13
@@ -60,8 +60,12 @@ def canonical_effective_config_document(cfg) -> dict:
|
|||||||
return {
|
return {
|
||||||
"schemaVersion": 1,
|
"schemaVersion": 1,
|
||||||
"dwh": dwh,
|
"dwh": dwh,
|
||||||
"vector": {"collection": collection, "dimensions": 1024, "distance": "cosine"},
|
"vector": {"collection": collection, "dimensions": int(embed_dim), "distance": "cosine"},
|
||||||
"embedding": {"model": model, "dimensions": int(embed_dim)},
|
"embedding": {
|
||||||
|
"id": getattr(embeddings, "id", None) or f"ollama/{model}",
|
||||||
|
"model": model,
|
||||||
|
"dimensions": int(embed_dim),
|
||||||
|
},
|
||||||
"roots": {
|
"roots": {
|
||||||
"artifacts": str(getattr(cfg.paths, "artifacts", Path("artifacts"))),
|
"artifacts": str(getattr(cfg.paths, "artifacts", Path("artifacts"))),
|
||||||
"indexes": str(getattr(cfg.paths, "indexes", Path("indexes"))),
|
"indexes": str(getattr(cfg.paths, "indexes", Path("indexes"))),
|
||||||
@@ -499,8 +503,9 @@ class EvidenceSourcesConfig(BaseModel):
|
|||||||
class EmbeddingsConfig(BaseModel):
|
class EmbeddingsConfig(BaseModel):
|
||||||
provider: str = "ollama_internal"
|
provider: str = "ollama_internal"
|
||||||
base_url: str
|
base_url: str
|
||||||
model: str = "nomic-embed-text-v2-moe"
|
id: str | None = None
|
||||||
dim: int = Field(default=768, alias="dimensions")
|
model: str
|
||||||
|
dim: int = Field(alias="dimensions")
|
||||||
batch_size: int = 16
|
batch_size: int = 16
|
||||||
timeout: int = 300
|
timeout: int = 300
|
||||||
connect_timeout: int = 5
|
connect_timeout: int = 5
|
||||||
@@ -642,6 +647,7 @@ def load_config(path: Path) -> Config:
|
|||||||
if not isinstance(raw, dict):
|
if not isinstance(raw, dict):
|
||||||
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
|
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
|
||||||
expanded = _resolve_secret_files(_resolve_evidence_secret_files(_expand_env(raw)))
|
expanded = _resolve_secret_files(_resolve_evidence_secret_files(_expand_env(raw)))
|
||||||
|
_apply_installation_embedding_projection(expanded, path)
|
||||||
_validate_internal_embedding_contract(expanded, path)
|
_validate_internal_embedding_contract(expanded, path)
|
||||||
_validate_internal_vector_contract(expanded, path)
|
_validate_internal_vector_contract(expanded, path)
|
||||||
translated, used_legacy = translate_legacy_config(expanded)
|
translated, used_legacy = translate_legacy_config(expanded)
|
||||||
@@ -710,6 +716,56 @@ def load_config(path: Path) -> Config:
|
|||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_installation_embedding_projection(raw: dict[str, Any], path: Path) -> None:
|
||||||
|
projected = {
|
||||||
|
"id": os.environ.get("THT_INTERNAL_EMBEDDING_ID"),
|
||||||
|
"model": os.environ.get("THT_INTERNAL_EMBEDDING_MODEL"),
|
||||||
|
"dimensions": os.environ.get("THT_INTERNAL_EMBEDDING_DIMENSIONS"),
|
||||||
|
}
|
||||||
|
if all(value is None for value in projected.values()):
|
||||||
|
return
|
||||||
|
if any(value is None for value in projected.values()):
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"la proiezione embedding dell'installazione è incompleta"
|
||||||
|
)
|
||||||
|
embedding_id = projected["id"]
|
||||||
|
model = projected["model"]
|
||||||
|
try:
|
||||||
|
dimensions = int(projected["dimensions"] or "")
|
||||||
|
except ValueError:
|
||||||
|
dimensions = 0
|
||||||
|
if embedding_id != f"ollama/{model}" or dimensions <= 0:
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"la proiezione embedding dell'installazione non è canonica"
|
||||||
|
)
|
||||||
|
|
||||||
|
sections: list[tuple[dict[str, Any], str]] = []
|
||||||
|
embeddings = raw.get("embeddings")
|
||||||
|
if isinstance(embeddings, dict):
|
||||||
|
sections.append((embeddings, "dim"))
|
||||||
|
resources = raw.get("resources")
|
||||||
|
if isinstance(resources, dict) and isinstance(resources.get("embeddings"), dict):
|
||||||
|
sections.append((resources["embeddings"], "dimensions"))
|
||||||
|
for section, dimension_key in sections:
|
||||||
|
declared = {
|
||||||
|
"id": section.get("id"),
|
||||||
|
"model": section.get("model"),
|
||||||
|
"dimensions": section.get(dimension_key),
|
||||||
|
}
|
||||||
|
expected = {"id": embedding_id, "model": model, "dimensions": dimensions}
|
||||||
|
for key, value in declared.items():
|
||||||
|
if value is not None and str(value) != str(expected[key]):
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
f"{key} è proprietà dell'installazione e diverge dalla proiezione attiva"
|
||||||
|
)
|
||||||
|
section["id"] = embedding_id
|
||||||
|
section["model"] = model
|
||||||
|
section[dimension_key] = dimensions
|
||||||
|
|
||||||
|
|
||||||
def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> None:
|
def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> None:
|
||||||
resources = raw.get("resources")
|
resources = raw.get("resources")
|
||||||
if not isinstance(resources, dict):
|
if not isinstance(resources, dict):
|
||||||
@@ -720,9 +776,10 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
|
|||||||
|
|
||||||
provider = embeddings.get("provider")
|
provider = embeddings.get("provider")
|
||||||
model = embeddings.get("model")
|
model = embeddings.get("model")
|
||||||
|
embedding_id = embeddings.get("id")
|
||||||
dimensions = embeddings.get("dimensions")
|
dimensions = embeddings.get("dimensions")
|
||||||
base_url = embeddings.get("base_url")
|
base_url = embeddings.get("base_url")
|
||||||
allowed = {"provider", "base_url", "model", "dimensions"}
|
allowed = {"provider", "base_url", "id", "model", "dimensions"}
|
||||||
unexpected = sorted(set(embeddings) - allowed)
|
unexpected = sorted(set(embeddings) - allowed)
|
||||||
if unexpected:
|
if unexpected:
|
||||||
raise ConfigError(
|
raise ConfigError(
|
||||||
@@ -734,15 +791,24 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
|
|||||||
f"Configurazione non valida in {path}:\n"
|
f"Configurazione non valida in {path}:\n"
|
||||||
"resources.embeddings.provider deve essere 'ollama_internal'"
|
"resources.embeddings.provider deve essere 'ollama_internal'"
|
||||||
)
|
)
|
||||||
if model != "qwen3-embedding:0.6b":
|
if not isinstance(model, str) or not model:
|
||||||
raise ConfigError(
|
raise ConfigError(
|
||||||
f"Configurazione non valida in {path}:\n"
|
f"Configurazione non valida in {path}:\n"
|
||||||
"resources.embeddings.model deve essere 'qwen3-embedding:0.6b'"
|
"resources.embeddings.model deve essere valorizzato"
|
||||||
)
|
)
|
||||||
if dimensions != 1024:
|
try:
|
||||||
|
parsed_dimensions = int(dimensions)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
parsed_dimensions = 0
|
||||||
|
if isinstance(dimensions, bool) or parsed_dimensions <= 0:
|
||||||
raise ConfigError(
|
raise ConfigError(
|
||||||
f"Configurazione non valida in {path}:\n"
|
f"Configurazione non valida in {path}:\n"
|
||||||
"resources.embeddings.dimensions deve essere 1024"
|
"resources.embeddings.dimensions deve essere un intero positivo"
|
||||||
|
)
|
||||||
|
if embedding_id is not None and embedding_id != f"ollama/{model}":
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"resources.embeddings.id deve essere l'identità canonica ollama/<model>"
|
||||||
)
|
)
|
||||||
if not _is_allowed_internal_embedding_url(base_url):
|
if not _is_allowed_internal_embedding_url(base_url):
|
||||||
raise ConfigError(
|
raise ConfigError(
|
||||||
@@ -805,15 +871,20 @@ def _validate_active_embeddings_config(
|
|||||||
f"Configurazione non valida in {path}:\n"
|
f"Configurazione non valida in {path}:\n"
|
||||||
"embeddings.provider deve essere 'ollama_internal'"
|
"embeddings.provider deve essere 'ollama_internal'"
|
||||||
)
|
)
|
||||||
if embeddings.model != "qwen3-embedding:0.6b":
|
if not embeddings.model:
|
||||||
raise ConfigError(
|
raise ConfigError(
|
||||||
f"Configurazione non valida in {path}:\n"
|
f"Configurazione non valida in {path}:\n"
|
||||||
"embeddings.model deve essere 'qwen3-embedding:0.6b'"
|
"embeddings.model deve essere valorizzato"
|
||||||
)
|
)
|
||||||
if embeddings.dim != 1024:
|
if embeddings.dim <= 0:
|
||||||
raise ConfigError(
|
raise ConfigError(
|
||||||
f"Configurazione non valida in {path}:\n"
|
f"Configurazione non valida in {path}:\n"
|
||||||
"embeddings.dim deve essere 1024"
|
"embeddings.dim deve essere un intero positivo"
|
||||||
|
)
|
||||||
|
if embeddings.id is not None and embeddings.id != f"ollama/{embeddings.model}":
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"embeddings.id deve essere l'identità canonica ollama/<model>"
|
||||||
)
|
)
|
||||||
if not _is_allowed_internal_embedding_url(embeddings.base_url):
|
if not _is_allowed_internal_embedding_url(embeddings.base_url):
|
||||||
raise ConfigError(
|
raise ConfigError(
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ from tht.evidence.contracts import (
|
|||||||
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
||||||
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
|
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||||
_EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$")
|
_EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||||
|
_CANONICAL_MODEL_ID = re.compile(
|
||||||
|
r"^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$"
|
||||||
|
)
|
||||||
_EVIDENCE_METADATA_KEYS = frozenset({
|
_EVIDENCE_METADATA_KEYS = frozenset({
|
||||||
"evidence_id", "evidence_kind", "purposes", "scope", "language", "provenance",
|
"evidence_id", "evidence_kind", "purposes", "scope", "language", "provenance",
|
||||||
})
|
})
|
||||||
@@ -37,6 +40,12 @@ def _validate_hash(value: str) -> str:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_canonical_model_id(value: str) -> str:
|
||||||
|
if not _CANONICAL_MODEL_ID.fullmatch(value):
|
||||||
|
raise ValueError("model identifier must use canonical provider/model form")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _require_content_hash(content: str, content_hash: str) -> None:
|
def _require_content_hash(content: str, content_hash: str) -> None:
|
||||||
expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
|
expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
|
||||||
if content_hash != expected:
|
if content_hash != expected:
|
||||||
@@ -146,6 +155,7 @@ class CorpusManifest(_WithMetadata):
|
|||||||
manifest_id: str | None = None
|
manifest_id: str | None = None
|
||||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||||
pipeline_version: str = Field(default="evidence-v1", min_length=1)
|
pipeline_version: str = Field(default="evidence-v1", min_length=1)
|
||||||
|
embedding_id: str | None = None
|
||||||
embedding_model: str | None = None
|
embedding_model: str | None = None
|
||||||
embedding_dimensions: int | None = Field(default=None, gt=0)
|
embedding_dimensions: int | None = Field(default=None, gt=0)
|
||||||
vector_generation: str | None = None
|
vector_generation: str | None = None
|
||||||
@@ -155,6 +165,9 @@ class CorpusManifest(_WithMetadata):
|
|||||||
_manifest_id = field_validator("manifest_id")(
|
_manifest_id = field_validator("manifest_id")(
|
||||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||||
)
|
)
|
||||||
|
_embedding_id = field_validator("embedding_id")(
|
||||||
|
lambda value: _validate_canonical_model_id(value) if value is not None else None
|
||||||
|
)
|
||||||
_vector_generation = field_validator("vector_generation")(
|
_vector_generation = field_validator("vector_generation")(
|
||||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||||
)
|
)
|
||||||
@@ -164,6 +177,10 @@ class CorpusManifest(_WithMetadata):
|
|||||||
def validate_generation(self) -> "CorpusManifest":
|
def validate_generation(self) -> "CorpusManifest":
|
||||||
if (self.embedding_model is None) != (self.embedding_dimensions is None):
|
if (self.embedding_model is None) != (self.embedding_dimensions is None):
|
||||||
raise ValueError("embedding_model and embedding_dimensions must be set together")
|
raise ValueError("embedding_model and embedding_dimensions must be set together")
|
||||||
|
if self.embedding_id is not None and self.embedding_model is None:
|
||||||
|
raise ValueError("embedding_id requires embedding model and dimensions")
|
||||||
|
if self.schema_version >= 2 and self.embedding_model is not None and self.embedding_id is None:
|
||||||
|
raise ValueError("schema version 2 embedding generations require embedding_id")
|
||||||
if self.vector_generation is not None and self.embedding_model is None:
|
if self.vector_generation is not None and self.embedding_model is None:
|
||||||
raise ValueError("vector_generation requires embedding model and dimension compatibility")
|
raise ValueError("vector_generation requires embedding model and dimension compatibility")
|
||||||
|
|
||||||
|
|||||||
@@ -125,6 +125,7 @@ class CorpusPipeline:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
|
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
|
||||||
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
|
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
|
||||||
|
embedding_id: str | None = None,
|
||||||
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
|
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
|
||||||
workspace_id: str | None = None, sparse_language: str = "italian",
|
workspace_id: str | None = None, sparse_language: str = "italian",
|
||||||
candidate_evaluator: Callable[[CorpusManifest], object] | None = None,
|
candidate_evaluator: Callable[[CorpusManifest], object] | None = None,
|
||||||
@@ -133,6 +134,7 @@ class CorpusPipeline:
|
|||||||
self.sources = sources
|
self.sources = sources
|
||||||
self.embedder = embedder
|
self.embedder = embedder
|
||||||
self.vector_store = vector_store
|
self.vector_store = vector_store
|
||||||
|
self.embedding_id = embedding_id or f"ollama/{embedding_model}"
|
||||||
self.embedding_model = embedding_model
|
self.embedding_model = embedding_model
|
||||||
self.embedding_dimensions = embedding_dimensions
|
self.embedding_dimensions = embedding_dimensions
|
||||||
self.chunk_policy = chunk_policy
|
self.chunk_policy = chunk_policy
|
||||||
@@ -284,6 +286,7 @@ class CorpusPipeline:
|
|||||||
source_by_id = {item.source_id: (source, item) for source, item in discovered}
|
source_by_id = {item.source_id: (source, item) for source, item in discovered}
|
||||||
compatibility = _fingerprint({
|
compatibility = _fingerprint({
|
||||||
"pipeline": self.pipeline_version,
|
"pipeline": self.pipeline_version,
|
||||||
|
"embedding_id": self.embedding_id,
|
||||||
"model": self.embedding_model,
|
"model": self.embedding_model,
|
||||||
"dimensions": self.embedding_dimensions,
|
"dimensions": self.embedding_dimensions,
|
||||||
"chunk_policy": asdict(self.chunk_policy),
|
"chunk_policy": asdict(self.chunk_policy),
|
||||||
@@ -294,6 +297,7 @@ class CorpusPipeline:
|
|||||||
"compatibility_fingerprint": compatibility,
|
"compatibility_fingerprint": compatibility,
|
||||||
"pipeline_version": self.pipeline_version,
|
"pipeline_version": self.pipeline_version,
|
||||||
"chunk_policy_version": self.chunk_policy.version,
|
"chunk_policy_version": self.chunk_policy.version,
|
||||||
|
"embedding_id": self.embedding_id,
|
||||||
"embedding_model": self.embedding_model,
|
"embedding_model": self.embedding_model,
|
||||||
"embedding_dimensions": self.embedding_dimensions,
|
"embedding_dimensions": self.embedding_dimensions,
|
||||||
}
|
}
|
||||||
@@ -492,7 +496,9 @@ class CorpusPipeline:
|
|||||||
) for document in documents
|
) for document in documents
|
||||||
}
|
}
|
||||||
manifest = CorpusManifest(
|
manifest = CorpusManifest(
|
||||||
|
schema_version=2,
|
||||||
pipeline_version=self.pipeline_version,
|
pipeline_version=self.pipeline_version,
|
||||||
|
embedding_id=self.embedding_id,
|
||||||
embedding_model=self.embedding_model,
|
embedding_model=self.embedding_model,
|
||||||
embedding_dimensions=self.embedding_dimensions,
|
embedding_dimensions=self.embedding_dimensions,
|
||||||
vector_generation=plan["generation"],
|
vector_generation=plan["generation"],
|
||||||
@@ -723,7 +729,8 @@ class CorpusPipeline:
|
|||||||
prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
||||||
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
|
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
|
||||||
compatibility = _fingerprint({
|
compatibility = _fingerprint({
|
||||||
"pipeline": self.pipeline_version, "model": self.embedding_model,
|
"pipeline": self.pipeline_version, "embedding_id": self.embedding_id,
|
||||||
|
"model": self.embedding_model,
|
||||||
"dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy),
|
"dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy),
|
||||||
})
|
})
|
||||||
previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None
|
previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None
|
||||||
@@ -761,7 +768,9 @@ class CorpusPipeline:
|
|||||||
for document in documents
|
for document in documents
|
||||||
}
|
}
|
||||||
manifest = CorpusManifest(
|
manifest = CorpusManifest(
|
||||||
|
schema_version=2,
|
||||||
pipeline_version=self.pipeline_version,
|
pipeline_version=self.pipeline_version,
|
||||||
|
embedding_id=self.embedding_id,
|
||||||
embedding_model=self.embedding_model,
|
embedding_model=self.embedding_model,
|
||||||
embedding_dimensions=self.embedding_dimensions,
|
embedding_dimensions=self.embedding_dimensions,
|
||||||
vector_generation=generation,
|
vector_generation=generation,
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ def build_preprocessing_pipeline(
|
|||||||
vector_store: VectorStore,
|
vector_store: VectorStore,
|
||||||
embedding_model: str,
|
embedding_model: str,
|
||||||
embedding_dimensions: int,
|
embedding_dimensions: int,
|
||||||
|
embedding_id: str | None = None,
|
||||||
chunk_policy: ChunkPolicy,
|
chunk_policy: ChunkPolicy,
|
||||||
pipeline_version: str,
|
pipeline_version: str,
|
||||||
retain_published_generations: int = 3,
|
retain_published_generations: int = 3,
|
||||||
@@ -35,6 +36,7 @@ def build_preprocessing_pipeline(
|
|||||||
sources=sources,
|
sources=sources,
|
||||||
embedder=embedder,
|
embedder=embedder,
|
||||||
vector_store=vector_store,
|
vector_store=vector_store,
|
||||||
|
embedding_id=embedding_id,
|
||||||
embedding_model=embedding_model,
|
embedding_model=embedding_model,
|
||||||
embedding_dimensions=embedding_dimensions,
|
embedding_dimensions=embedding_dimensions,
|
||||||
chunk_policy=chunk_policy,
|
chunk_policy=chunk_policy,
|
||||||
|
|||||||
@@ -39,8 +39,6 @@ evidence:
|
|||||||
|
|
||||||
embeddings:
|
embeddings:
|
||||||
base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434
|
base_url: ${THT_OLLAMA_URL} # http://host.docker.internal:11434
|
||||||
model: nomic-embed-text-v2-moe
|
|
||||||
dim: 768
|
|
||||||
batch_size: 32
|
batch_size: 32
|
||||||
|
|
||||||
# Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che
|
# Vector: diretto (read+write). L'assenza di vector_rest/vector_write_rest fa sì che
|
||||||
|
|||||||
@@ -25,8 +25,6 @@ paths:
|
|||||||
|
|
||||||
embeddings:
|
embeddings:
|
||||||
base_url: ${THT_OLLAMA_URL}
|
base_url: ${THT_OLLAMA_URL}
|
||||||
model: nomic-embed-text-v2-moe
|
|
||||||
dim: 768
|
|
||||||
batch_size: 64
|
batch_size: 64
|
||||||
|
|
||||||
# LOADING diretto del pgvector (server-only). Su workstation la lettura passa da
|
# LOADING diretto del pgvector (server-only). Su workstation la lettura passa da
|
||||||
|
|||||||
@@ -39,8 +39,6 @@ evidence:
|
|||||||
|
|
||||||
embeddings:
|
embeddings:
|
||||||
base_url: ${THT_OLLAMA_URL} # es. http://localhost:11434
|
base_url: ${THT_OLLAMA_URL} # es. http://localhost:11434
|
||||||
model: nomic-embed-text-v2-moe
|
|
||||||
dim: 768
|
|
||||||
batch_size: 32
|
batch_size: 32
|
||||||
|
|
||||||
vectors:
|
vectors:
|
||||||
|
|||||||
@@ -528,8 +528,13 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
|
|||||||
defer func() { _ = lock.Release() }()
|
defer func() { _ = lock.Release() }()
|
||||||
}
|
}
|
||||||
if args[0] == "restart" || args[0] == "update" || args[0] == "rollback" {
|
if args[0] == "restart" || args[0] == "update" || args[0] == "rollback" {
|
||||||
if err := modelprojection.Generate(installation); err != nil {
|
drift, err := modelprojection.Check(installation)
|
||||||
return commandUsageError(stderr, "installation model catalog could not be projected: "+err.Error())
|
if err != nil {
|
||||||
|
return commandUsageError(stderr, "installation model catalog could not be verified: "+err.Error())
|
||||||
|
}
|
||||||
|
if len(drift) > 0 {
|
||||||
|
return commandUsageError(stderr,
|
||||||
|
"installation model catalog changed; run tht start to apply the complete runtime projection")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
|
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
|
||||||
|
|||||||
@@ -1057,6 +1057,7 @@ func TestRunPiStatusUsesImageBundledPi(t *testing.T) {
|
|||||||
func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T) {
|
func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T) {
|
||||||
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
||||||
fixture.setEnvironment(t)
|
fixture.setEnvironment(t)
|
||||||
|
fixture.generateModelProjection(t)
|
||||||
if err := os.WriteFile(filepath.Join(fixture.projectDirectory, "docker", "core.Dockerfile"), []byte("ARG PI_VERSION=0.79.0\n"), 0o600); err != nil {
|
if err := os.WriteFile(filepath.Join(fixture.projectDirectory, "docker", "core.Dockerfile"), []byte("ARG PI_VERSION=0.79.0\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1079,6 +1080,7 @@ func TestRunPiUpdateResolvesLatestStableVersionWhenVersionIsOmitted(t *testing.T
|
|||||||
func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
|
func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
|
||||||
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
||||||
fixture.setEnvironment(t)
|
fixture.setEnvironment(t)
|
||||||
|
fixture.generateModelProjection(t)
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
exitCode := run(context.Background(), []string{"--installation", fixture.installationPath, "pi", "update", "--version", "0.81.0", "--source", "build"}, &stdout, &stderr)
|
exitCode := run(context.Background(), []string{"--installation", fixture.installationPath, "pi", "update", "--version", "0.81.0", "--source", "build"}, &stdout, &stderr)
|
||||||
@@ -1095,6 +1097,7 @@ func TestRunPiUpdateRequiresExplicitConfirmationWithoutInvokingDocker(t *testing
|
|||||||
func TestRunPiRestartRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
|
func TestRunPiRestartRequiresExplicitConfirmationWithoutInvokingDocker(t *testing.T) {
|
||||||
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
||||||
fixture.setEnvironment(t)
|
fixture.setEnvironment(t)
|
||||||
|
fixture.generateModelProjection(t)
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
exitCode := run(context.Background(), []string{
|
exitCode := run(context.Background(), []string{
|
||||||
@@ -1117,6 +1120,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
fixture.setEnvironment(t, secretPath)
|
fixture.setEnvironment(t, secretPath)
|
||||||
|
fixture.generateModelProjection(t)
|
||||||
t.Setenv("THT_FAKE_PI_VERSION", "pi-restart-secret")
|
t.Setenv("THT_FAKE_PI_VERSION", "pi-restart-secret")
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
@@ -1138,6 +1142,7 @@ func TestRunPiRestartSanitizesSuccessOutput(t *testing.T) {
|
|||||||
func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) {
|
func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) {
|
||||||
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
||||||
fixture.setEnvironment(t)
|
fixture.setEnvironment(t)
|
||||||
|
fixture.generateModelProjection(t)
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
exitCode := run(context.Background(), []string{
|
exitCode := run(context.Background(), []string{
|
||||||
@@ -1152,6 +1157,21 @@ func TestRunPiUpdateAcceptsExplicitVersionWithoutAdvancedFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPiRestartRejectsCatalogDriftBeforeInvokingDocker(t *testing.T) {
|
||||||
|
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
||||||
|
fixture.setEnvironment(t)
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
exitCode := run(context.Background(), []string{
|
||||||
|
"--installation", fixture.installationPath, "pi", "restart", "--yes",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if exitCode != 2 || !strings.Contains(stderr.String(), "run tht start") {
|
||||||
|
t.Fatalf("run() = %d, stderr=%q; want complete projection guidance", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
assertDockerNotInvoked(t, fixture)
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPiMaintenanceStatusAndRecoverConfirmationContract(t *testing.T) {
|
func TestRunPiMaintenanceStatusAndRecoverConfirmationContract(t *testing.T) {
|
||||||
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
fixture := newCLIFixture(t, "THT_LLM_URL=https://llm.example.invalid\n")
|
||||||
fixture.setEnvironment(t)
|
fixture.setEnvironment(t)
|
||||||
|
|||||||
@@ -78,6 +78,13 @@ func TestLoadRejectsInvalidModelCatalogDefaultsAndAuthentication(t *testing.T) {
|
|||||||
catalog: strings.Replace(validModelCatalogYAML(), "id: ollama/qwen3-embedding:0.6b", "id: external/qwen3-embedding:0.6b", 1),
|
catalog: strings.Replace(validModelCatalogYAML(), "id: ollama/qwen3-embedding:0.6b", "id: external/qwen3-embedding:0.6b", 1),
|
||||||
wantError: "ollama canonical id",
|
wantError: "ollama canonical id",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "upstream model aliases must be unique within a provider",
|
||||||
|
catalog: strings.Replace(validModelCatalogYAML(),
|
||||||
|
" models:\n glm-5.3:\n",
|
||||||
|
" models:\n glm-alias:\n upstreamModel: glm-5.3\n session:\n contextWindow: 200000\n maxTokens: 131072\n glm-5.3:\n", 1),
|
||||||
|
wantError: "same upstreamModel",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "metadata default required",
|
name: "metadata default required",
|
||||||
catalog: strings.Replace(validModelCatalogYAML(), " metadataGeneration: local/qwen\n", "", 1),
|
catalog: strings.Replace(validModelCatalogYAML(), " metadataGeneration: local/qwen\n", "", 1),
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ const maxCatalogModels = 64
|
|||||||
|
|
||||||
var catalogKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}$`)
|
var catalogKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}$`)
|
||||||
var catalogModelIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`)
|
var catalogModelIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`)
|
||||||
|
var catalogUpstreamModelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`)
|
||||||
var catalogAPIVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
|
var catalogAPIVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
|
||||||
|
|
||||||
// ModelCatalog is the only operator-authored source for model identity and runtime eligibility.
|
// ModelCatalog is the only operator-authored source for model identity and runtime eligibility.
|
||||||
@@ -163,11 +164,23 @@ func (c ModelCatalog) Validate(environment map[string]string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
hasSession, providerHasMetadata := false, false
|
hasSession, providerHasMetadata := false, false
|
||||||
|
upstreamModels := make(map[string]string)
|
||||||
for modelID, model := range provider.Models {
|
for modelID, model := range provider.Models {
|
||||||
if !catalogKeyPattern.MatchString(modelID) {
|
if !catalogKeyPattern.MatchString(modelID) {
|
||||||
return fmt.Errorf("modelCatalog model %q/%q is invalid", providerID, modelID)
|
return fmt.Errorf("modelCatalog model %q/%q is invalid", providerID, modelID)
|
||||||
}
|
}
|
||||||
canonical := providerID + "/" + modelID
|
canonical := providerID + "/" + modelID
|
||||||
|
upstream := model.UpstreamModel
|
||||||
|
if upstream == "" {
|
||||||
|
upstream = modelID
|
||||||
|
}
|
||||||
|
if !catalogUpstreamModelPattern.MatchString(upstream) {
|
||||||
|
return fmt.Errorf("modelCatalog model %q upstreamModel is invalid", canonical)
|
||||||
|
}
|
||||||
|
if existing, duplicate := upstreamModels[upstream]; duplicate {
|
||||||
|
return fmt.Errorf("modelCatalog models %q and %q use the same upstreamModel", existing, canonical)
|
||||||
|
}
|
||||||
|
upstreamModels[upstream] = canonical
|
||||||
if model.Session == nil && model.MetadataGeneration == nil {
|
if model.Session == nil && model.MetadataGeneration == nil {
|
||||||
return fmt.Errorf("modelCatalog model %q has no runtime use", canonical)
|
return fmt.Errorf("modelCatalog model %q has no runtime use", canonical)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -119,9 +119,10 @@ func Run(request Request) error {
|
|||||||
if !filepath.IsAbs(request.InstallationPath) || filepath.Base(request.InstallationPath) != "thothii-installation.yaml" {
|
if !filepath.IsAbs(request.InstallationPath) || filepath.Base(request.InstallationPath) != "thothii-installation.yaml" {
|
||||||
return errors.New("installation migration requires an absolute thothii-installation.yaml path")
|
return errors.New("installation migration requires an absolute thothii-installation.yaml path")
|
||||||
}
|
}
|
||||||
if !filepath.IsAbs(request.OutputPath) || filepath.Base(request.OutputPath) != "thothii-installation.yaml" ||
|
outputExtension := strings.ToLower(filepath.Ext(request.OutputPath))
|
||||||
|
if !filepath.IsAbs(request.OutputPath) || outputExtension != ".yaml" && outputExtension != ".yml" ||
|
||||||
filepath.Clean(request.OutputPath) == filepath.Clean(request.InstallationPath) {
|
filepath.Clean(request.OutputPath) == filepath.Clean(request.InstallationPath) {
|
||||||
return errors.New("installation migration output must be a different absolute thothii-installation.yaml path")
|
return errors.New("installation migration output must be a different absolute YAML path")
|
||||||
}
|
}
|
||||||
if request.SessionDefault == "" || request.EmbeddingID == "" || request.EmbeddingDimensions <= 0 {
|
if request.SessionDefault == "" || request.EmbeddingID == "" || request.EmbeddingDimensions <= 0 {
|
||||||
return errors.New("installation migration requires session default, embedding id, and positive embedding dimensions")
|
return errors.New("installation migration requires session default, embedding id, and positive embedding dimensions")
|
||||||
|
|||||||
@@ -69,6 +69,19 @@ func TestRunReportsProviderAuthenticationConflictAndPublishesNothing(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunAcceptsVersionedYAMLCandidateFilename(t *testing.T) {
|
||||||
|
fixture := migrationFixture(t, false)
|
||||||
|
request := fixture.request()
|
||||||
|
request.OutputPath = filepath.Join(filepath.Dir(request.OutputPath), "thothii-installation.v2.yaml")
|
||||||
|
|
||||||
|
if err := Run(request); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if info, err := os.Stat(request.OutputPath); err != nil || !info.Mode().IsRegular() {
|
||||||
|
t.Fatalf("versioned candidate: info=%v err=%v", info, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type migrationTestFixture struct {
|
type migrationTestFixture struct {
|
||||||
legacy string
|
legacy string
|
||||||
output string
|
output string
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ const (
|
|||||||
ComposeFile = "compose.models.yaml"
|
ComposeFile = "compose.models.yaml"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var renameProjectionDirectory = os.Rename
|
||||||
|
|
||||||
type runtimeCatalog struct {
|
type runtimeCatalog struct {
|
||||||
SchemaVersion int `json:"schemaVersion"`
|
SchemaVersion int `json:"schemaVersion"`
|
||||||
DefaultSession string `json:"defaultSession"`
|
DefaultSession string `json:"defaultSession"`
|
||||||
@@ -87,10 +89,11 @@ func Render(installation config.Installation) (map[string][]byte, error) {
|
|||||||
if model.Session == nil {
|
if model.Session == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
enabled = append(enabled, model.ID)
|
|
||||||
if model.SessionAdapter == nil || model.SessionAdapter.Mode != "openai_compatible" {
|
if model.SessionAdapter == nil || model.SessionAdapter.Mode != "openai_compatible" {
|
||||||
|
enabled = append(enabled, model.ID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
enabled = append(enabled, model.Provider+"/"+model.UpstreamModel)
|
||||||
provider := customProviders[model.Provider]
|
provider := customProviders[model.Provider]
|
||||||
provider.BaseURL = model.Endpoint.BaseURL
|
provider.BaseURL = model.Endpoint.BaseURL
|
||||||
provider.API = "openai-completions"
|
provider.API = "openai-completions"
|
||||||
@@ -137,52 +140,84 @@ func Render(installation config.Installation) (map[string][]byte, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate atomically replaces each generated adapter after the complete candidate has rendered.
|
// Generate publishes all adapters as one directory generation. A failed replacement restores the
|
||||||
|
// previous directory, so callers never observe a successfully returned mixed generation.
|
||||||
func Generate(installation config.Installation) error {
|
func Generate(installation config.Installation) error {
|
||||||
artifacts, err := Render(installation)
|
artifacts, err := Render(installation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
paths := sortedArtifactPaths(artifacts)
|
target := installation.GeneratedDirectory()
|
||||||
for _, relative := range paths {
|
parent := filepath.Dir(target)
|
||||||
destination := filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(relative))
|
if err := os.MkdirAll(parent, 0o755); err != nil {
|
||||||
// The generated catalog contains references to secret environment variable names, never
|
return fmt.Errorf("create model projection parent: %w", err)
|
||||||
// secret values. Core runs as an unprivileged container user and must be able to traverse
|
}
|
||||||
// the bind-mounted host directories and read the projections.
|
candidate, err := os.MkdirTemp(parent, ".model-projections-candidate-*")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create model projection candidate: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = os.RemoveAll(candidate) }()
|
||||||
|
if err := writeProjectionCandidate(candidate, artifacts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
info, statErr := os.Lstat(target)
|
||||||
|
if os.IsNotExist(statErr) {
|
||||||
|
if err := renameProjectionDirectory(candidate, target); err != nil {
|
||||||
|
return fmt.Errorf("publish model projection generation: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if statErr != nil {
|
||||||
|
return fmt.Errorf("inspect current model projection generation: %w", statErr)
|
||||||
|
}
|
||||||
|
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("current model projection path is not a regular directory")
|
||||||
|
}
|
||||||
|
previous, err := absentTemporaryPath(parent)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reserve previous model projection generation: %w", err)
|
||||||
|
}
|
||||||
|
if err := renameProjectionDirectory(target, previous); err != nil {
|
||||||
|
return fmt.Errorf("prepare model projection generation replacement: %w", err)
|
||||||
|
}
|
||||||
|
if err := renameProjectionDirectory(candidate, target); err != nil {
|
||||||
|
if restoreErr := renameProjectionDirectory(previous, target); restoreErr != nil {
|
||||||
|
return fmt.Errorf("publish model projection generation: %v; restore previous generation: %w", err, restoreErr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("publish model projection generation: %w", err)
|
||||||
|
}
|
||||||
|
_ = os.RemoveAll(previous)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeProjectionCandidate(directory string, artifacts map[string][]byte) error {
|
||||||
|
if err := os.Chmod(directory, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("protect model projection candidate: %w", err)
|
||||||
|
}
|
||||||
|
for _, relative := range sortedArtifactPaths(artifacts) {
|
||||||
|
destination := filepath.Join(directory, filepath.FromSlash(relative))
|
||||||
if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
|
||||||
return fmt.Errorf("create model projection directory: %w", err)
|
return fmt.Errorf("create model projection directory: %w", err)
|
||||||
}
|
}
|
||||||
if err := os.Chmod(filepath.Dir(destination), 0o755); err != nil {
|
if err := os.WriteFile(destination, artifacts[relative], 0o644); err != nil {
|
||||||
return fmt.Errorf("protect model projection directory: %w", err)
|
|
||||||
}
|
|
||||||
temporary, err := os.CreateTemp(filepath.Dir(destination), ".projection-*")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create model projection candidate: %w", err)
|
|
||||||
}
|
|
||||||
temporaryName := temporary.Name()
|
|
||||||
published := false
|
|
||||||
defer func() {
|
|
||||||
if !published {
|
|
||||||
_ = os.Remove(temporaryName)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if err := temporary.Chmod(0o644); err == nil {
|
|
||||||
_, err = temporary.Write(artifacts[relative])
|
|
||||||
}
|
|
||||||
if closeErr := temporary.Close(); err == nil {
|
|
||||||
err = closeErr
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("write model projection candidate: %w", err)
|
return fmt.Errorf("write model projection candidate: %w", err)
|
||||||
}
|
}
|
||||||
if err := os.Rename(temporaryName, destination); err != nil {
|
|
||||||
return fmt.Errorf("publish model projection: %w", err)
|
|
||||||
}
|
|
||||||
published = true
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func absentTemporaryPath(parent string) (string, error) {
|
||||||
|
path, err := os.MkdirTemp(parent, ".model-projections-previous-*")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Check returns relative artifact names whose current bytes differ from the catalog projection.
|
// Check returns relative artifact names whose current bytes differ from the catalog projection.
|
||||||
func Check(installation config.Installation) ([]string, error) {
|
func Check(installation config.Installation) ([]string, error) {
|
||||||
artifacts, err := Render(installation)
|
artifacts, err := Render(installation)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package modelprojection
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -41,9 +42,12 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T)
|
|||||||
t.Fatalf("Pi models projection must contain only custom providers:\n%s", piModels)
|
t.Fatalf("Pi models projection must contain only custom providers:\n%s", piModels)
|
||||||
}
|
}
|
||||||
settings := string(first[PiSettingsFile])
|
settings := string(first[PiSettingsFile])
|
||||||
if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen"`) {
|
if !strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen2.5:7b"`) {
|
||||||
t.Fatalf("Pi settings projection missing enabled session models:\n%s", settings)
|
t.Fatalf("Pi settings projection missing enabled session models:\n%s", settings)
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(piModels, `"id": "qwen2.5:7b"`) || !strings.Contains(catalog, `"id": "local/qwen"`) {
|
||||||
|
t.Fatalf("Pi projection must map canonical catalog keys to upstream model IDs")
|
||||||
|
}
|
||||||
compose := string(first[ComposeFile])
|
compose := string(first[ComposeFile])
|
||||||
for _, expected := range []string{"THT_MODEL_CATALOG_FILE", "THT_DEFAULT_SESSION_MODEL", "zai/glm-5.3", "THT_INTERNAL_EMBEDDING_ID", "THT_INTERNAL_EMBEDDING_MODEL", "OLLAMA_MODEL", installation.GeneratedModelCatalogPath()} {
|
for _, expected := range []string{"THT_MODEL_CATALOG_FILE", "THT_DEFAULT_SESSION_MODEL", "zai/glm-5.3", "THT_INTERNAL_EMBEDDING_ID", "THT_INTERNAL_EMBEDDING_MODEL", "OLLAMA_MODEL", installation.GeneratedModelCatalogPath()} {
|
||||||
if !strings.Contains(compose, expected) {
|
if !strings.Contains(compose, expected) {
|
||||||
@@ -52,6 +56,46 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGenerateRestoresWholePreviousGenerationWhenPublishFails(t *testing.T) {
|
||||||
|
installation := projectionFixture(t)
|
||||||
|
if err := Generate(installation); err != nil {
|
||||||
|
t.Fatalf("Generate() initial error = %v", err)
|
||||||
|
}
|
||||||
|
before := make(map[string][]byte)
|
||||||
|
for _, path := range []string{CatalogFile, PiModelsFile, PiSettingsFile, ComposeFile} {
|
||||||
|
contents, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before[path] = contents
|
||||||
|
}
|
||||||
|
provider := installation.ModelCatalog.Providers["zai"]
|
||||||
|
model := provider.Models["glm-5.3"]
|
||||||
|
model.Label = "Changed label"
|
||||||
|
provider.Models["glm-5.3"] = model
|
||||||
|
installation.ModelCatalog.Providers["zai"] = provider
|
||||||
|
|
||||||
|
originalRename := renameProjectionDirectory
|
||||||
|
t.Cleanup(func() { renameProjectionDirectory = originalRename })
|
||||||
|
renames := 0
|
||||||
|
renameProjectionDirectory = func(oldPath, newPath string) error {
|
||||||
|
renames++
|
||||||
|
if renames == 2 {
|
||||||
|
return errors.New("injected publish failure")
|
||||||
|
}
|
||||||
|
return os.Rename(oldPath, newPath)
|
||||||
|
}
|
||||||
|
if err := Generate(installation); err == nil {
|
||||||
|
t.Fatal("Generate() error = nil, want injected publish failure")
|
||||||
|
}
|
||||||
|
for path, expected := range before {
|
||||||
|
actual, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
|
||||||
|
if err != nil || !bytes.Equal(actual, expected) {
|
||||||
|
t.Fatalf("artifact %q changed after failed generation: err=%v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGeneratePublishesAllArtifactsAndCheckDetectsDrift(t *testing.T) {
|
func TestGeneratePublishesAllArtifactsAndCheckDetectsDrift(t *testing.T) {
|
||||||
installation := projectionFixture(t)
|
installation := projectionFixture(t)
|
||||||
if err := Generate(installation); err != nil {
|
if err := Generate(installation); err != nil {
|
||||||
@@ -84,7 +128,7 @@ func projectionFixture(t *testing.T) config.Installation {
|
|||||||
Providers: map[string]config.ModelProvider{
|
Providers: map[string]config.ModelProvider{
|
||||||
"deepseek": {Authentication: config.ModelAuthentication{Mode: "pi_auth"}, Session: &config.ModelSessionAdapter{Mode: "pi_builtin"}, Models: map[string]config.CatalogModel{"deepseek-v4-pro": {Session: &config.SessionModel{}}}},
|
"deepseek": {Authentication: config.ModelAuthentication{Mode: "pi_auth"}, Session: &config.ModelSessionAdapter{Mode: "pi_builtin"}, Models: map[string]config.CatalogModel{"deepseek-v4-pro": {Session: &config.SessionModel{}}}},
|
||||||
"zai": {Endpoint: &config.ModelEndpoint{BaseURL: "https://api.z.ai/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, Models: map[string]config.CatalogModel{"glm-5.3": {Label: "GLM-5.3", Session: &config.SessionModel{Reasoning: true, ContextWindow: 200000, MaxTokens: 131072}}}},
|
"zai": {Endpoint: &config.ModelEndpoint{BaseURL: "https://api.z.ai/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, Models: map[string]config.CatalogModel{"glm-5.3": {Label: "GLM-5.3", Session: &config.SessionModel{Reasoning: true, ContextWindow: 200000, MaxTokens: 131072}}}},
|
||||||
"local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}},
|
"local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {UpstreamModel: "qwen2.5:7b", Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user