mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: unify installation model catalog

Browse Source
This commit is contained in:
Codex
2026-09-02 18:45:33 +02:00
parent ae053961a3
commit 7b7927bfe5
169 changed files with 3696 additions and 4572 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/app.ts
+19 -7
View File
@@ -22,7 +22,8 @@ import { isUsableAuthenticationSecret } from "./auth/secret-policy.js";
import { secretValue } from "./config/secret-bundle.js";
import { sessionRoutes } from "./routes/sessions.js";
import { sqlRoutes } from "./routes/sql.js";
import { metaRoutes, type ListModelsFn } from "./routes/meta.js";
import { metaRoutes } from "./routes/meta.js";
import type { ListModelsFn } from "./pi/list-models.js";
import { settingsRoutes, effectiveSettings } from "./routes/settings.js";
import { createPiModelLister } from "./pi/list-models.js";
import { createPiManagement, type PiManagementService } from "./pi/management.js";
@@ -66,6 +67,7 @@ import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description
import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js";
import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
import { loadRuntimeModelCatalog, type RuntimeModelCatalog } from "./models/runtime-model-catalog.js";
export interface BuildAppDeps {
thtRunner?: ThtRunner;
@@ -88,6 +90,7 @@ export interface BuildAppDeps {
catalogSyncWorker?: CatalogSyncWorker;
catalogOperationCoordinator?: CatalogOperationCoordinator;
metadataGenerationModels?: MetadataGenerationModels;
runtimeModelCatalog?: RuntimeModelCatalog;
modelCompleter?: ModelCompleter;
descriptionSourceSampler?: DescriptionSourceSampler;
workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
@@ -155,17 +158,22 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
internalEmbeddingId: config.internalEmbeddingId,
internalEmbeddingModel: config.internalEmbeddingModel,
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
},
});
const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
const hub = deps?.hub ?? new SseHub();
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
const runtimeModelCatalog = deps?.runtimeModelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
const mgr = deps?.mgr ?? new PiProcessManager(config, {
...(deps?.spawnFn ? { spawnFn: deps.spawnFn } : {}),
modelCatalog: runtimeModelCatalog,
});
const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
installationFile: config.installationConfigFile,
catalogFile: config.modelCatalogFile,
secretsFile: config.secretsFile,
});
const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
@@ -237,6 +245,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
internalEmbeddingId: config.internalEmbeddingId,
internalEmbeddingModel: config.internalEmbeddingModel,
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
});
@@ -271,7 +280,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
const getSettings = async (principal: PrincipalContext): Promise<Settings> => {
if (deps?.getSettings) return await deps.getSettings(principal);
const stored = loadSettings(config);
const effective = effectiveSettings(config, stored);
const effective = effectiveSettings(config, stored, runtimeModelCatalog);
// In the registry system the legacy `harness/workspaces/*.yaml` default is obsolete: when no
// installation workspace is pinned, default to the first active registry workspace.
if (!stored.workspace) {
@@ -284,7 +293,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
}
return effective;
};
const piManagement = deps?.piManagement ?? createPiManagement(config, { listModels });
const piManagement = deps?.piManagement ?? createPiManagement(config, {
modelCatalog: runtimeModelCatalog,
});
const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile);
const localRegistryResolver = deps?.localUserRegistry === undefined
@@ -408,6 +419,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
dwhPrecheck: config.dwhPrecheck,
legacyWorkspaceMode: config.legacyWorkspaceMode,
workspaceRuntimeSupport,
modelCatalog: runtimeModelCatalog,
maintenanceBarrier,
effectiveRelationships,
});
@@ -439,7 +451,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
return maintenanceBarrier.status();
});
sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
metaRoutes(app, { harnessDir: config.harnessDir, listModels });
metaRoutes(app, { harnessDir: config.harnessDir, modelCatalog: runtimeModelCatalog });
workspaceRoutes(app, {
registry: workspaceRegistry,
config: config.workspaceRegistry,
@@ -472,7 +484,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
worker: descriptionGenerationWorker,
sensitiveDataSuggestionRunner,
});
settingsRoutes(app, { cfg: config, listModels, getSettings });
settingsRoutes(app, { cfg: config, getSettings });
piManagementRoutes(app, { service: piManagement });
return app;
backend/src/catalog/metadata-generation-models.ts
+41 -162
View File
@@ -1,63 +1,5 @@
import {
closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
type Stats,
} from "node:fs";
import { parseAllDocuments } from "yaml";
import { z } from "zod";
import {
loadSecretBundle,
METADATA_GENERATION_SECRET_KEYS,
} from "../config/secret-bundle.js";
const MAX_INSTALLATION_BYTES = 1024 * 1024;
const RUNTIME_INSTALLATION_FILE = "/run/thothii-installation/thothii-installation.yaml";
const modelId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
const apiKeyEnvironment = z.enum(METADATA_GENERATION_SECRET_KEYS);
const endpointSchema = z.object({
baseUrl: z.string().min(1).max(2048).refine((value) => {
try {
const url = new URL(value);
return (url.protocol === "http:" || url.protocol === "https:")
&& url.username === "" && url.password === "" && url.search === "" && url.hash === "";
} catch {
return false;
}
}),
apiVersion: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/).optional(),
}).strict();
const configuredModelSchema = z.object({
id: modelId,
label: z.string().min(1).max(128).refine((value) => value.trim() === value && !/\p{Cc}/u.test(value)),
litellm: z.object({
provider: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/),
model: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/),
disableThinking: z.literal(true).optional(),
endpoint: endpointSchema.optional(),
}).strict(),
apiKeyEnv: apiKeyEnvironment.optional(),
}).strict().superRefine((value, context) => {
if (value.apiKeyEnv === undefined && value.litellm.endpoint === undefined) {
context.addIssue({
code: z.ZodIssueCode.custom,
path: ["apiKeyEnv"],
message: "keyless models require an explicit endpoint",
});
}
if (value.litellm.disableThinking === true && value.litellm.endpoint === undefined) {
context.addIssue({
code: z.ZodIssueCode.custom,
path: ["litellm", "disableThinking"],
message: "thinking may be disabled only for an explicit endpoint",
});
}
});
const metadataGenerationSchema = z.object({
default: modelId.optional(),
models: z.array(configuredModelSchema).max(64).default([]),
}).strict();
const installationSchema = z.object({
metadataGeneration: metadataGenerationSchema.optional(),
}).passthrough();
import { loadSecretBundle } from "../config/secret-bundle.js";
import { loadRuntimeModelCatalog } from "../models/runtime-model-catalog.js";
export interface MetadataGenerationModelChoice {
id: string;
@@ -86,7 +28,6 @@ export class MetadataGenerationModelUnavailableError extends Error {
}
}
/** The complete interface callers need: safe discovery plus fail-closed runtime resolution. */
export interface MetadataGenerationModels {
catalog(): MetadataGenerationModelCatalog;
resolve(selection: string): ResolvedMetadataGenerationModel;
@@ -96,140 +37,78 @@ class RestartLoadedMetadataGenerationModels implements MetadataGenerationModels
readonly #models: ReadonlyMap<string, ResolvedMetadataGenerationModel>;
readonly #catalog: MetadataGenerationModelCatalog;
constructor(
models: ReadonlyMap<string, ResolvedMetadataGenerationModel> = new Map(),
defaultModel: string | null = null,
choices: MetadataGenerationModelChoice[] = [],
) {
constructor(models: ReadonlyMap<string, ResolvedMetadataGenerationModel>, defaultModel: string | null) {
this.#models = models;
this.#catalog = {
models: choices.map((choice) => ({ ...choice })),
models: [...models.values()].map(({ id }) => ({ id, label: id })),
default: defaultModel,
};
}
catalog(): MetadataGenerationModelCatalog {
return {
models: this.#catalog.models.map((choice) => ({ ...choice })),
default: this.#catalog.default,
};
return { models: this.#catalog.models.map((choice) => ({ ...choice })), default: this.#catalog.default };
}
resolve(selection: string): ResolvedMetadataGenerationModel {
const model = typeof selection === "string" ? this.#models.get(selection) : undefined;
const model = this.#models.get(selection);
if (!model) throw new MetadataGenerationModelUnavailableError();
return model;
}
}
function invalid(message = "metadata-generation configuration is invalid"): Error {
function invalid(message = "metadata-generation runtime catalog is invalid"): Error {
return new Error(message);
}
function protectedInstallationStat(file: string, info: Stats): boolean {
const mode = info.mode & 0o777;
if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
|| info.size < 1 || info.size > MAX_INSTALLATION_BYTES) return false;
if (file === RUNTIME_INSTALLATION_FILE && info.uid === 0 && mode === 0o444) return true;
return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600);
}
function readProtectedInstallation(file: string): string {
let descriptor: number | undefined;
try {
const before = lstatSync(file);
if (!protectedInstallationStat(file, before)) throw new Error("unavailable");
descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
const opened = fstatSync(descriptor);
if (!protectedInstallationStat(file, opened)
|| before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("unavailable");
const source = readFileSync(descriptor, "utf8");
const after = fstatSync(descriptor);
const current = lstatSync(file);
if (!protectedInstallationStat(file, after) || !protectedInstallationStat(file, current)
|| opened.dev !== after.dev || opened.ino !== after.ino
|| opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("unavailable");
return source;
} finally {
if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized below */ }
}
}
function readInstallation(file: string): unknown {
try {
const documents = parseAllDocuments(readProtectedInstallation(file), { uniqueKeys: true });
if (documents.length !== 1) throw invalid("metadata-generation installation must contain one YAML document");
const document = documents[0];
if (document.errors.length > 0 || document.warnings.length > 0) {
throw invalid("metadata-generation installation contains invalid YAML");
}
return document.toJSON();
} catch (error) {
if (error instanceof Error && error.message.startsWith("metadata-generation")) throw error;
throw invalid("metadata-generation installation is unavailable");
}
}
export function loadMetadataGenerationModels(options: {
installationFile?: string;
catalogFile?: string;
secretsFile?: string;
}): MetadataGenerationModels {
if (!options.installationFile) return new RestartLoadedMetadataGenerationModels();
const installation = installationSchema.safeParse(readInstallation(options.installationFile));
if (!installation.success) throw invalid();
const configured = installation.data.metadataGeneration;
if (!configured || configured.models.length === 0) {
if (configured?.default !== undefined) throw invalid("metadata-generation default does not identify a configured model");
return new RestartLoadedMetadataGenerationModels();
}
if (!configured.default) throw invalid("metadata-generation default is required when models are configured");
const catalog = loadRuntimeModelCatalog(options.catalogFile);
const configured = catalog.metadataModels();
if (configured.length === 0) return new RestartLoadedMetadataGenerationModels(new Map(), null);
const seen = new Set<string>();
for (const model of configured.models) {
if (seen.has(model.id)) throw invalid(`metadata-generation model id "${model.id}" is duplicated`);
seen.add(model.id);
}
if (!seen.has(configured.default)) {
throw invalid(`metadata-generation default "${configured.default}" is not configured`);
}
const requiresSecrets = configured.models.some((model) => model.apiKeyEnv !== undefined);
const requiresSecrets = configured.some((model) => model.authentication.mode === "secret_env");
let secrets: ReadonlyMap<string, string> = new Map();
if (requiresSecrets) {
if (!options.secretsFile) throw invalid("metadata-generation keyed models require THT_SECRETS_FILE");
try {
secrets = loadSecretBundle(options.secretsFile);
} catch {
throw invalid("metadata-generation secrets are unavailable");
}
try { secrets = loadSecretBundle(options.secretsFile); }
catch { throw invalid("metadata-generation secrets are unavailable"); }
}
const models = new Map<string, ResolvedMetadataGenerationModel>();
for (const configuredModel of configured.models) {
const labels = new Map<string, string>();
for (const configuredModel of configured) {
const adapter = configuredModel.metadataAdapter;
if (!adapter || configuredModel.authentication.mode === "pi_auth") throw invalid();
const apiKeyEnv = configuredModel.authentication.apiKeyEnv;
let apiKey: string | undefined;
if (configuredModel.apiKeyEnv !== undefined) {
apiKey = secrets.get(configuredModel.apiKeyEnv);
if (!apiKey) {
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is missing`);
}
if (configuredModel.authentication.mode === "secret_env") {
if (!apiKeyEnv) throw invalid();
apiKey = secrets.get(apiKeyEnv);
if (!apiKey) throw invalid(`metadata-generation model "${configuredModel.id}" secret "${apiKeyEnv}" is missing`);
if (apiKey.length > 16 * 1024 || /\s/u.test(apiKey)) {
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is unusable`);
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${apiKeyEnv}" is unusable`);
}
}
labels.set(configuredModel.id, configuredModel.label);
models.set(configuredModel.id, Object.freeze({
id: configuredModel.id,
provider: configuredModel.litellm.provider,
model: configuredModel.litellm.model,
...(configuredModel.litellm.disableThinking === true ? { disableThinking: true as const } : {}),
...(configuredModel.litellm.endpoint === undefined
? {}
: { endpoint: Object.freeze({ ...configuredModel.litellm.endpoint }) }),
...(configuredModel.apiKeyEnv === undefined
? {}
: { apiKeyEnv: configuredModel.apiKeyEnv, apiKey }),
provider: adapter.litellmProvider,
model: configuredModel.upstreamModel,
...(configuredModel.metadataGeneration?.disableThinking === true
? { disableThinking: true as const } : {}),
...(configuredModel.endpoint ? { endpoint: Object.freeze({ ...configuredModel.endpoint }) } : {}),
...(apiKeyEnv ? { apiKeyEnv, apiKey } : {}),
}));
}
return new RestartLoadedMetadataGenerationModels(
models,
configured.default,
configured.models.map(({ id, label }) => ({ id, label })),
);
const result = new RestartLoadedMetadataGenerationModels(models, catalog.defaultMetadataGeneration);
const safe = result.catalog();
return {
catalog: () => ({
default: safe.default,
models: safe.models.map((choice) => ({ ...choice, label: labels.get(choice.id) ?? choice.id })),
}),
resolve: (selection) => result.resolve(selection),
};
}
backend/src/catalog/migrate.ts
+2
View File
@@ -12,6 +12,7 @@ import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_fla
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
import * as canonicalModelIdsMigration from "./migrations/010_canonical_model_ids.js";
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
const host = process.env.THT_CATALOG_DB_HOST;
@@ -46,6 +47,7 @@ const provider: MigrationProvider = {
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
"009_ai_token_usage": aiTokenUsageMigration,
"010_canonical_model_ids": canonicalModelIdsMigration,
};
},
};
backend/src/catalog/migrations/010_canonical_model_ids.ts
+27
View File
@@ -0,0 +1,27 @@
import { sql, type Kysely } from "kysely";
import type { CatalogDatabase } from "../repository.js";
const canonicalModelPattern = "^[a-z][a-z0-9._-]{0,63}/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$";
const legacyModelPattern = "^[a-z][a-z0-9._-]{0,63}$";
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
await sql.raw(`alter table description_generation_runs
drop constraint description_generation_runs_model_id_check,
add constraint description_generation_runs_model_id_check
check (model_id ~ '${canonicalModelPattern}')`).execute(db);
await sql.raw(`alter table sensitive_data_suggestion_runs
drop constraint sensitive_data_suggestion_runs_model_id_check,
add constraint sensitive_data_suggestion_runs_model_id_check
check (model_id ~ '${canonicalModelPattern}')`).execute(db);
}
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
await sql.raw(`alter table sensitive_data_suggestion_runs
drop constraint sensitive_data_suggestion_runs_model_id_check,
add constraint sensitive_data_suggestion_runs_model_id_check
check (model_id ~ '${legacyModelPattern}')`).execute(db);
await sql.raw(`alter table description_generation_runs
drop constraint description_generation_runs_model_id_check,
add constraint description_generation_runs_model_id_check
check (model_id ~ '${legacyModelPattern}')`).execute(db);
}
backend/src/config.ts
+32 -2
View File
@@ -32,6 +32,7 @@ export interface AppConfig {
piManagementTimeoutMs: number;
secretsFile?: string;
installationConfigFile?: string;
modelCatalogFile?: string;
piAuthFile?: string;
secretFiles: Readonly<Record<string, string | undefined>>;
modelApiKeyFile?: string;
@@ -50,6 +51,7 @@ export interface AppConfig {
workspaceSecretRuntimeRoot: string;
internalQdrantUrl: string;
internalEmbeddingUrl: string;
internalEmbeddingId: string;
internalEmbeddingModel: string;
internalEmbeddingDimensions: number;
}
@@ -189,6 +191,21 @@ function positiveDimension(value: string | undefined, fallback: number): number
return parsed;
}
function internalEmbeddingIdentity(
identityValue: string | undefined,
modelValue: string | undefined,
): { id: string; model: string } {
const id = identityValue ?? "ollama/qwen3-embedding:0.6b";
if (!/^ollama\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/.test(id)) {
throw new Error("internal embedding identity configuration is invalid");
}
const model = id.slice(id.indexOf("/") + 1);
if (modelValue !== undefined && modelValue !== model) {
throw new Error("internal embedding model does not match its canonical identity");
}
return { id, model };
}
function catalogDatabase(env: Record<string, string | undefined>): CatalogConnectionConfig | undefined {
const value = env.THT_CATALOG_DATABASE_URL;
if (value !== undefined) {
@@ -330,6 +347,13 @@ export function loadConfig(
|| installationConfigFile.includes("\0")
|| !path.isAbsolute(installationConfigFile)
)) throw new Error("installation configuration is invalid");
const modelCatalogFile = env.THT_MODEL_CATALOG_FILE;
if (modelCatalogFile !== undefined && (
modelCatalogFile.trim() !== modelCatalogFile
|| modelCatalogFile.length === 0
|| modelCatalogFile.includes("\0")
|| !path.isAbsolute(modelCatalogFile)
)) throw new Error("runtime model catalog configuration is invalid");
const piAuthFile = env.THT_PI_AUTH_FILE;
if (piAuthFile !== undefined && (
piAuthFile.trim() !== piAuthFile || piAuthFile.length === 0 || piAuthFile.includes("\0")
@@ -391,6 +415,10 @@ export function loadConfig(
"internal embedding URL",
["embedding", "localhost"],
);
const internalEmbedding = internalEmbeddingIdentity(
env.THT_INTERNAL_EMBEDDING_ID,
env.THT_INTERNAL_EMBEDDING_MODEL,
);
return {
host: env.HOST ?? "127.0.0.1",
port: Number(env.PORT ?? 8787),
@@ -404,7 +432,7 @@ export function loadConfig(
publicExposure,
sessionStorage,
catalogDatabase: catalogDatabase(env),
defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING },
defaults: { thinking: env.PI_THINKING },
maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4),
settingsFile,
maintenanceFile: env.THT_MAINTENANCE_FILE ?? path.join(path.dirname(settingsFile), "maintenance.json"),
@@ -413,6 +441,7 @@ export function loadConfig(
piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
secretsFile,
installationConfigFile,
modelCatalogFile,
piAuthFile,
secretFiles,
modelApiKeyFile,
@@ -425,7 +454,8 @@ export function loadConfig(
workspaceSecretRuntimeRoot,
internalQdrantUrl,
internalEmbeddingUrl,
internalEmbeddingModel: env.THT_INTERNAL_EMBEDDING_MODEL ?? "qwen3-embedding:0.6b",
internalEmbeddingId: internalEmbedding.id,
internalEmbeddingModel: internalEmbedding.model,
internalEmbeddingDimensions: positiveDimension(env.THT_INTERNAL_EMBEDDING_DIMENSIONS, 1024),
};
}
backend/src/config/secret-bundle.ts
+1 -1
View File
@@ -8,7 +8,7 @@ import {
isUsableAuthenticationSecret,
} from "../auth/secret-policy.js";
/** Credential names that metadata-generation model entries may reference. */
/** Credential names that Installation Model Catalog providers may reference. */
export const METADATA_GENERATION_SECRET_KEYS = Object.freeze([
"THT_METADATA_API_KEY", "ANTHROPIC_API_KEY", "AZURE_API_KEY", "GEMINI_API_KEY",
"DEEPSEEK_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ZAI_API_KEY",
backend/src/models/runtime-model-catalog.ts
+161
View File
@@ -0,0 +1,161 @@
import {
closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, type Stats,
} from "node:fs";
import { z } from "zod";
const MAX_CATALOG_BYTES = 1024 * 1024;
const RUNTIME_CATALOG_FILE = "/run/thothii-model-catalog/catalog.json";
const canonicalId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/);
const secretBundleKey = /^[A-Z][A-Z0-9_]{0,63}$/;
const endpointSchema = z.object({
baseUrl: z.string().url(),
apiVersion: z.string().optional(),
}).strict();
const authenticationSchema = z.object({
mode: z.enum(["secret_env", "pi_auth", "none"]),
apiKeyEnv: z.string().optional(),
}).strict();
const runtimeModelSchema = z.object({
id: canonicalId,
provider: z.string().min(1),
model: z.string().min(1),
label: z.string().min(1),
upstreamModel: z.string().min(1),
endpoint: endpointSchema.optional(),
authentication: authenticationSchema,
sessionAdapter: z.object({ mode: z.enum(["pi_builtin", "openai_compatible"]) }).strict().optional(),
metadataAdapter: z.object({ litellmProvider: z.string().min(1) }).strict().optional(),
session: z.object({
reasoning: z.boolean(),
input: z.array(z.string()).optional(),
cost: z.object({
input: z.number(), output: z.number(), cacheRead: z.number(), cacheWrite: z.number(),
}).strict().optional(),
contextWindow: z.number().int().positive().optional(),
maxTokens: z.number().int().positive().optional(),
compatibility: z.object({
supportsDeveloperRole: z.boolean(),
supportsReasoningEffort: z.boolean(),
supportsStore: z.boolean(),
maxTokensField: z.string().optional(),
}).strict().optional(),
}).strict().optional(),
metadataGeneration: z.object({ disableThinking: z.boolean() }).strict().optional(),
}).strict();
const catalogSchema = z.object({
schemaVersion: z.literal(1),
defaultSession: canonicalId,
defaultMetadataGeneration: canonicalId.optional(),
embedding: z.object({ id: canonicalId, dimensions: z.number().int().positive() }).strict(),
models: z.array(runtimeModelSchema).max(64),
}).strict();
export type RuntimeModel = z.infer<typeof runtimeModelSchema>;
export interface RuntimeModelCatalog {
readonly defaultSession: string | null;
readonly defaultMetadataGeneration: string | null;
readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
sessionModels(): readonly RuntimeModel[];
metadataModels(): readonly RuntimeModel[];
hasSession(id: string): boolean;
}
class RestartLoadedRuntimeModelCatalog implements RuntimeModelCatalog {
readonly defaultSession: string | null;
readonly defaultMetadataGeneration: string | null;
readonly embedding: Readonly<{ id: string; dimensions: number }> | null;
readonly #sessions: readonly RuntimeModel[];
readonly #metadata: readonly RuntimeModel[];
readonly #sessionIds: ReadonlySet<string>;
constructor(catalog?: z.infer<typeof catalogSchema>) {
this.defaultSession = catalog?.defaultSession ?? null;
this.defaultMetadataGeneration = catalog?.defaultMetadataGeneration ?? null;
this.embedding = catalog ? Object.freeze({ ...catalog.embedding }) : null;
this.#sessions = Object.freeze((catalog?.models ?? []).filter((model) => model.session !== undefined));
this.#metadata = Object.freeze((catalog?.models ?? []).filter((model) => model.metadataGeneration !== undefined));
this.#sessionIds = new Set(this.#sessions.map((model) => model.id));
}
sessionModels(): readonly RuntimeModel[] { return this.#sessions.map((model) => ({ ...model })); }
metadataModels(): readonly RuntimeModel[] { return this.#metadata.map((model) => ({ ...model })); }
hasSession(id: string): boolean { return this.#sessionIds.has(id); }
}
function protectedCatalogStat(file: string, info: Stats): boolean {
const mode = info.mode & 0o777;
if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
|| info.size < 1 || info.size > MAX_CATALOG_BYTES) return false;
if (file === RUNTIME_CATALOG_FILE && info.uid === 0 && (mode === 0o444 || mode === 0o644)) return true;
return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600 || mode === 0o644);
}
function readProtectedCatalog(file: string): unknown {
let descriptor: number | undefined;
try {
const before = lstatSync(file);
if (!protectedCatalogStat(file, before)) throw new Error("runtime model catalog is unavailable");
descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
const opened = fstatSync(descriptor);
if (!protectedCatalogStat(file, opened)
|| before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("runtime model catalog is unavailable");
const source = readFileSync(descriptor, "utf8");
const after = fstatSync(descriptor);
const current = lstatSync(file);
if (!protectedCatalogStat(file, after) || !protectedCatalogStat(file, current)
|| opened.dev !== after.dev || opened.ino !== after.ino
|| opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("runtime model catalog is unavailable");
return JSON.parse(source);
} catch {
throw new Error("runtime model catalog is unavailable");
} finally {
if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized above */ }
}
}
export function loadRuntimeModelCatalog(file?: string): RuntimeModelCatalog {
if (!file) return new RestartLoadedRuntimeModelCatalog();
const parsed = catalogSchema.safeParse(readProtectedCatalog(file));
if (!parsed.success) throw new Error("runtime model catalog is invalid");
if (parsed.data.models.some((model) => !validRuntimeModel(model))) {
throw new Error("runtime model catalog is invalid");
}
const ids = new Set(parsed.data.models.map((model) => model.id));
if (ids.size !== parsed.data.models.length) throw new Error("runtime model catalog contains duplicate models");
const sessions = parsed.data.models.filter((model) => model.session !== undefined).map((model) => model.id);
const metadata = parsed.data.models.filter((model) => model.metadataGeneration !== undefined).map((model) => model.id);
if (!sessions.includes(parsed.data.defaultSession)) throw new Error("runtime model catalog session default is invalid");
if ((metadata.length > 0) !== (parsed.data.defaultMetadataGeneration !== undefined)
|| (parsed.data.defaultMetadataGeneration !== undefined
&& !metadata.includes(parsed.data.defaultMetadataGeneration))) {
throw new Error("runtime model catalog metadata default is invalid");
}
return new RestartLoadedRuntimeModelCatalog(parsed.data);
}
function validRuntimeModel(model: RuntimeModel): boolean {
if ((model.session !== undefined) !== (model.sessionAdapter !== undefined)) return false;
if ((model.metadataGeneration !== undefined) !== (model.metadataAdapter !== undefined)) return false;
switch (model.authentication.mode) {
case "secret_env":
return model.authentication.apiKeyEnv !== undefined
&& secretBundleKey.test(model.authentication.apiKeyEnv);
case "pi_auth":
return model.authentication.apiKeyEnv === undefined
&& model.metadataGeneration === undefined
&& model.sessionAdapter?.mode === "pi_builtin";
case "none":
return model.authentication.apiKeyEnv === undefined && model.endpoint !== undefined;
}
}
export function splitCanonicalModelId(id: string): { provider: string; model: string } {
const slash = id.indexOf("/");
if (slash <= 0 || slash === id.length - 1) throw new Error("model identity is invalid");
return { provider: id.slice(0, slash), model: id.slice(slash + 1) };
}
backend/src/operator-command.ts
+8 -6
View File
@@ -8,8 +8,8 @@ import { join } from "node:path";
import { loadConfig, type AppConfig } from "./config.js";
import { rolesToPermissions } from "./auth/config.js";
import type { PrincipalContext } from "./auth/principal.js";
import { createPiModelLister } from "./pi/list-models.js";
import { createPiManagement } from "./pi/management.js";
import { loadRuntimeModelCatalog } from "./models/runtime-model-catalog.js";
import { effectiveSettings } from "./routes/settings.js";
import { MaintenanceBarrier } from "./runtime/maintenance-gate.js";
import { loadSettings } from "./settings/settings-store.js";
@@ -19,7 +19,7 @@ import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status"
| "session-inventory" | "workflow-doctor" | "workspace-integrity"
| "pi-options" | "pi-test" | "effective-settings";
| "pi-test" | "effective-settings";
const lifecyclePrincipal: PrincipalContext = {
issuer: "tht-operator-command",
@@ -110,9 +110,11 @@ export async function runOperatorAction(
if (action === "session-inventory") return await sessionInventory(config);
if (action === "workflow-doctor") return await workflowDiagnostics(config);
if (action === "workspace-integrity") return await workspaceIntegrity(config);
if (action === "effective-settings") return effectiveSettings(config, loadSettings(config));
const service = createPiManagement(config, { listModels: createPiModelLister(config) });
if (action === "pi-options") return await service.options();
const modelCatalog = loadRuntimeModelCatalog(config.modelCatalogFile);
if (action === "effective-settings") {
return effectiveSettings(config, loadSettings(config), modelCatalog);
}
const service = createPiManagement(config, { modelCatalog });
if (action === "pi-test") return await service.test();
throw new Error("unsupported operator action");
}
@@ -121,7 +123,7 @@ async function main(): Promise<void> {
const action = process.argv[2] as OperatorAction | undefined;
if (!action || ![
"maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory",
"workflow-doctor", "workspace-integrity", "pi-options", "pi-test", "effective-settings",
"workflow-doctor", "workspace-integrity", "pi-test", "effective-settings",
].includes(action)) throw new Error("invalid operator action");
const result = await runOperatorAction(action, loadConfig(process.env));
process.stdout.write(`${JSON.stringify(result)}\n`);
backend/src/pi/list-models.ts
+3 -1
View File
@@ -18,6 +18,8 @@ export interface PiModel {
reasoning: boolean;
}
export type ListModelsFn = () => Promise<PiModel[]>;
interface Opts {
spawnFn?: (
command: string,
@@ -36,7 +38,7 @@ interface Opts {
* configured) via an ephemeral `pi --mode rpc` process. Result is cached for
* `ttlMs`. The returned function rejects on timeout/error; callers degrade.
*/
export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): () => Promise<PiModel[]> {
export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): ListModelsFn {
const ttlMs = opts.ttlMs ?? 60_000;
const now = opts.nowMs ?? (() => Date.now());
const spawnFn = opts.spawnFn ?? nodeSpawn;
backend/src/pi/management.ts
+27 -92
View File
@@ -2,12 +2,11 @@ import { execFile as nodeExecFile } from "node:child_process";
import { promisify } from "node:util";
import type { AppConfig } from "../config.js";
import { secretValue } from "../config/secret-bundle.js";
import { loadSettings, type Settings } from "../settings/settings-store.js";
import {
loadSettings,
saveSettings,
type Settings,
} from "../settings/settings-store.js";
import type { PiModel } from "./list-models.js";
splitCanonicalModelId,
type RuntimeModelCatalog,
} from "../models/runtime-model-catalog.js";
import {
configuredPiProviderApiKey,
PI_MANAGED_CONFIG_ERROR_MESSAGE,
@@ -45,13 +44,6 @@ export interface PiStatus {
message?: string;
}
export interface PiOptions {
providers: string[];
models: Array<{ provider: string; id: string }>;
reasoning: PiReasoning[];
checkedAt: string;
}
export interface PiTestResult {
ready: boolean;
checkedAt: string;
@@ -76,15 +68,13 @@ export type PiExecFile = (
export interface PiManagementService {
status(): Promise<PiStatus>;
options(): Promise<PiOptions>;
configure(value: PiInstallationConfig): Promise<PiInstallationConfig & { updatedAt: string }>;
test(): Promise<PiTestResult>;
logs(): Promise<PiLogs>;
}
export class PiManagementError extends Error {
constructor(
public readonly code: "pi_management_invalid_config" | "pi_management_unavailable" | "pi_management_write_failed",
public readonly code: "pi_management_unavailable",
message: string,
) {
super(message);
@@ -93,10 +83,9 @@ export class PiManagementError extends Error {
interface PiManagementDeps {
execute?: PiExecFile;
listModels: () => Promise<PiModel[]>;
modelCatalog: RuntimeModelCatalog;
smokeProvider?: PiProviderSmoke;
readSettings?: () => Settings;
saveSettings?: (settings: Settings) => Settings;
readLogs?: () => string | Promise<string>;
credentialStatus?: (provider: string | undefined) => PiCredentialStatus;
now?: () => Date;
@@ -111,54 +100,36 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
};
const execute = deps.execute ?? defaultExecFile;
const readSettings = deps.readSettings ?? (() => loadSettings(config));
const persistSettings = deps.saveSettings ?? ((settings) => saveSettings(config, settings));
const readLogs = deps.readLogs ?? (() => diagnostics.join("\n"));
const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config);
const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config, {
modelCatalog: deps.modelCatalog,
});
const credentialStatus = deps.credentialStatus ?? ((provider: string | undefined) => {
try {
const model = deps.modelCatalog.defaultSession
? deps.modelCatalog.sessionModels().find((entry) => entry.id === deps.modelCatalog.defaultSession)
: undefined;
const credentialName = model?.authentication.mode === "secret_env"
? model.authentication.apiKeyEnv
: undefined;
const configuredApiKey = configuredPiProviderApiKey(
readConfiguredPiAgentFile("models.json", true),
provider,
) ?? (credentialName ? `$${credentialName}` : undefined);
return piProviderCredentialStatus({
provider,
authProviders: loadPiAuthProviders(),
resolveCredentialValue: () => secretValue(config, "THT_MODEL_API_KEY"),
resolveCredentialValue: () => credentialName
? secretValue(config, credentialName)
: config.modelCatalogFile ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
credentialFile: config.modelApiKeyFile,
configuredApiKey: configuredPiProviderApiKey(
readConfiguredPiAgentFile("models.json", true),
provider,
),
configuredApiKey,
});
} catch {
return "missing";
}
});
const closedOptions = async (): Promise<Omit<PiOptions, "checkedAt">> => {
let listed: PiModel[];
try {
listed = await deps.listModels();
} catch (error) {
if (isPiManagedConfigError(error)) {
throw new PiManagementError("pi_management_unavailable", PI_MANAGED_CONFIG_ERROR_MESSAGE);
}
throw new PiManagementError("pi_management_unavailable", "Pi model choices are unavailable");
}
const models: Array<{ provider: string; id: string }> = [];
const providers: string[] = [];
const seenModels = new Set<string>();
const seenProviders = new Set<string>();
for (const model of listed) {
if (!isChoice(model?.provider) || !isChoice(model?.id)) continue;
const key = `${model.provider}\u0000${model.id}`;
if (seenModels.has(key)) continue;
seenModels.add(key);
models.push({ provider: model.provider, id: model.id });
if (!seenProviders.has(model.provider)) {
seenProviders.add(model.provider);
providers.push(model.provider);
}
}
return { providers, models, reasoning: [...REASONING_CHOICES] };
};
const version = async (timeoutMs = config.piManagementTimeoutMs): Promise<string> => {
let output: { stdout: string; stderr: string };
try {
@@ -180,12 +151,12 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
const installationConfig = (): PiInstallationConfig => {
const settings = readSettings();
const provider = config.defaults.provider ?? settings.provider;
const model = config.defaults.model ?? settings.model;
const reasoning = config.defaults.thinking ?? settings.thinking;
const selected = deps.modelCatalog.defaultSession
? splitCanonicalModelId(deps.modelCatalog.defaultSession)
: undefined;
return {
...(isChoice(provider) ? { provider } : {}),
...(isChoice(model) ? { model } : {}),
...(selected ? selected : {}),
...(isReasoning(reasoning) ? { reasoning } : {}),
};
};
@@ -206,28 +177,6 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
}
},
async options(): Promise<PiOptions> {
const choices = await closedOptions();
return { ...choices, checkedAt: now().toISOString() };
},
async configure(value: PiInstallationConfig): Promise<PiInstallationConfig & { updatedAt: string }> {
if (!isInstallationConfig(value)) {
throw new PiManagementError("pi_management_invalid_config", "Pi installation configuration is invalid");
}
const choices = await closedOptions();
if (!choices.models.some((model) => model.provider === value.provider && model.id === value.model)) {
throw new PiManagementError("pi_management_invalid_config", "Pi provider and model must be selected from available choices");
}
try {
persistSettings({ ...readSettings(), provider: value.provider, model: value.model, thinking: value.reasoning });
} catch {
throw new PiManagementError("pi_management_write_failed", "Pi installation configuration could not be saved");
}
addDiagnostic("Pi installation defaults updated");
return { ...value, updatedAt: now().toISOString() };
},
async test(): Promise<PiTestResult> {
const checkedAt = now().toISOString();
const deadline = Date.now() + config.piManagementTimeoutMs;
@@ -293,24 +242,10 @@ async function defaultExecFile(command: string, args: string[], options: PiExecF
return { stdout: String(result.stdout), stderr: String(result.stderr) };
}
function isChoice(value: unknown): value is string {
return typeof value === "string" && value.length > 0 && value.length <= 128 && value.trim() === value
&& /^[A-Za-z0-9][A-Za-z0-9._/-]*$/u.test(value);
}
function isReasoning(value: unknown): value is PiReasoning {
return typeof value === "string" && (REASONING_CHOICES as readonly string[]).includes(value);
}
function isInstallationConfig(value: unknown): value is Required<PiInstallationConfig> {
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const candidate = value as Record<string, unknown>;
if (Object.keys(candidate).length !== 3 || Object.keys(candidate).some((key) => !["provider", "model", "reasoning"].includes(key))) {
return false;
}
return isChoice(candidate.provider) && isChoice(candidate.model) && isReasoning(candidate.reasoning);
}
function isTimeout(error: unknown): boolean {
return Boolean(
error && typeof error === "object" && (
backend/src/pi/pi-process-manager.ts
+36 -11
View File
@@ -11,6 +11,10 @@ import {
configuredPiProviderApiKey,
createPiRuntimeAgentSnapshot,
} from "./managed-config.js";
import {
loadRuntimeModelCatalog,
type RuntimeModelCatalog,
} from "../models/runtime-model-catalog.js";
export interface SessionRuntime {
rpc: RpcClient;
@@ -42,23 +46,32 @@ export class PiProcessManager {
private runtimes = new Map<string, SessionRuntime>();
private agentSnapshotCleanups = new WeakMap<ChildProcessWithoutNullStreams, () => void>();
private spawnFn: (
sessionId: string, author: string, provider: string | undefined, principal?: PrincipalContext,
runtimeConfigPath?: string,
sessionId: string, author: string, provider: string | undefined, model: string | undefined,
principal?: PrincipalContext, runtimeConfigPath?: string,
) => ChildProcessWithoutNullStreams;
private loadAuthProviders: (agentDir: string) => ReadonlySet<string>;
private modelCatalog: RuntimeModelCatalog;
private modelCatalogConfigured: boolean;
constructor(
private cfg: AppConfig,
opts?: { spawnFn?: SpawnFn; authProviders?: (agentDir: string) => ReadonlySet<string> },
opts?: {
spawnFn?: SpawnFn;
authProviders?: (agentDir: string) => ReadonlySet<string>;
modelCatalog?: RuntimeModelCatalog;
},
) {
this.modelCatalog = opts?.modelCatalog ?? loadRuntimeModelCatalog(cfg.modelCatalogFile);
this.modelCatalogConfigured = cfg.modelCatalogFile !== undefined
|| this.modelCatalog.defaultSession !== null;
this.loadAuthProviders = opts?.authProviders
?? ((agentDir) => loadPiAuthProviders({ agentDir }));
if (opts?.spawnFn) {
this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) =>
this.spawnPi(opts.spawnFn!, sessionId, author, provider, principal, runtimeConfigPath);
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
this.spawnPi(opts.spawnFn!, sessionId, author, provider, model, principal, runtimeConfigPath);
} else {
this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) =>
this.spawnPi(nodeSpawn, sessionId, author, provider, principal, runtimeConfigPath);
this.spawnFn = (sessionId, author, provider, model, principal, runtimeConfigPath) =>
this.spawnPi(nodeSpawn, sessionId, author, provider, model, principal, runtimeConfigPath);
}
}
@@ -71,7 +84,7 @@ export class PiProcessManager {
private spawnPi(
spawnFn: SpawnFn, sessionId: string, author: string, provider: string | undefined,
principal?: PrincipalContext, runtimeConfigPath?: string,
model: string | undefined, principal?: PrincipalContext, runtimeConfigPath?: string,
): ChildProcessWithoutNullStreams {
// This is the final shared boundary for createFor(), spawnFor(), and resume(). Validate
// before auth-provider inspection, then make Pi consume the exact copied bytes rather than
@@ -79,12 +92,23 @@ export class PiProcessManager {
const agent = createPiRuntimeAgentSnapshot();
let child: ChildProcessWithoutNullStreams | undefined;
try {
const catalogModel = provider && model
? this.modelCatalog.sessionModels()
.find((entry) => entry.provider === provider && entry.model === model)
: undefined;
const credentialName = catalogModel?.authentication.mode === "secret_env"
? catalogModel.authentication.apiKeyEnv
: undefined;
const projectedApiKey = configuredPiProviderApiKey(agent.models, provider)
?? (credentialName ? `$${credentialName}` : undefined);
const env = buildPiChildEnv({
provider,
authProviders: this.loadAuthProviders(agent.agentDir),
credentialValue: secretValue(this.cfg, "THT_MODEL_API_KEY"),
credentialValue: credentialName
? secretValue(this.cfg, credentialName)
: this.modelCatalogConfigured ? undefined : secretValue(this.cfg, "THT_MODEL_API_KEY"),
credentialFile: this.cfg.modelApiKeyFile,
configuredApiKey: configuredPiProviderApiKey(agent.models, provider),
configuredApiKey: projectedApiKey,
additions: { THT_SESSION: sessionId, THT_AUTHOR: author },
});
env.PI_CODING_AGENT_DIR = agent.agentDir;
@@ -162,9 +186,10 @@ export class PiProcessManager {
}
const author = o.author ?? "dev@local";
const provider = canonicalPiProvider(o.provider ?? this.cfg.defaults.provider);
const model = o.model ?? this.cfg.defaults.model;
let child: ChildProcessWithoutNullStreams;
try {
child = this.spawnFn(sessionId, author, provider, o.principal, o.runtimeConfig?.path);
child = this.spawnFn(sessionId, author, provider, model, o.principal, o.runtimeConfig?.path);
} catch (error) {
o.runtimeConfig?.release();
throw error;
backend/src/pi/provider-smoke.ts
+19 -2
View File
@@ -17,6 +17,10 @@ import {
readConfiguredPiAgentFile,
validateDeclarativePiConfig,
} from "./managed-config.js";
import {
loadRuntimeModelCatalog,
type RuntimeModelCatalog,
} from "../models/runtime-model-catalog.js";
const SMOKE_PROMPT = "Provider health check. Reply with exactly OK.";
const SMOKE_ARGS = [
@@ -49,6 +53,7 @@ interface ProviderSmokeOptions {
authProviders?: () => ReadonlySet<string>;
readAuthStore?: () => string;
readModelsStore?: () => string | undefined;
modelCatalog?: RuntimeModelCatalog;
}
export function createPiProviderSmoke(
@@ -69,12 +74,24 @@ export function createPiProviderSmoke(
const configuredModels = options.readModelsStore
? options.readModelsStore()
: readConfiguredPiAgentFile("models.json", true);
const catalog = options.modelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
const catalogConfigured = config.modelCatalogFile !== undefined
|| catalog.defaultSession !== null;
const catalogModel = catalog.sessionModels()
.find((entry) => entry.provider === canonicalProvider && entry.model === model);
const credentialName = catalogModel?.authentication.mode === "secret_env"
? catalogModel.authentication.apiKeyEnv
: undefined;
const projectedApiKey = configuredPiProviderApiKey(configuredModels, canonicalProvider)
?? (credentialName ? `$${credentialName}` : undefined);
const env = buildPiChildEnv({
provider: canonicalProvider,
authProviders: configuredAuthProviders,
credentialValue: secretValue(config, "THT_MODEL_API_KEY"),
credentialValue: credentialName
? secretValue(config, credentialName)
: catalogConfigured ? undefined : secretValue(config, "THT_MODEL_API_KEY"),
credentialFile: config.modelApiKeyFile,
configuredApiKey: configuredPiProviderApiKey(configuredModels, canonicalProvider),
configuredApiKey: projectedApiKey,
});
clearPrincipalEnvironment(env);
delete env.THT_DATA_ROOT;
backend/src/routes/catalog-description-generation.ts
+1 -1
View File
@@ -32,7 +32,7 @@ import {
} from "../catalog/types.js";
const idSchema = z.uuid();
const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/);
const selectedTargetIdsSchema = z.array(idSchema).min(1);
const suggestionSchema = z.discriminatedUnion("scope", [
z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
backend/src/routes/meta.ts
+10 -15
View File
@@ -1,10 +1,8 @@
import { readdirSync } from "node:fs";
import { join } from "node:path";
import type { FastifyInstance } from "fastify";
import type { PiModel } from "../pi/list-models.js";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
export type ListModelsFn = () => Promise<PiModel[]>;
import type { RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
/**
* List YAML workspace configs found in <harnessDir>/workspaces/*.yaml.
@@ -25,20 +23,17 @@ export function listWorkspaces(harnessDir: string): { name: string; file: string
export function metaRoutes(
app: FastifyInstance,
deps: { harnessDir: string; listModels?: ListModelsFn },
deps: { harnessDir: string; modelCatalog: RuntimeModelCatalog },
): void {
app.get("/models", async (request, reply) => {
if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply;
const fn = deps.listModels ?? (async () => []);
try {
return { models: await fn() };
} catch (error) {
app.log.warn({
component: "pi-model-list",
errorType: error instanceof Error ? error.name : typeof error,
}, "Pi model listing failed");
// Graceful fallback: Pi may not be running; don't crash the server.
return { models: [] as PiModel[] };
}
return {
models: deps.modelCatalog.sessionModels().map((entry) => ({
provider: entry.provider,
id: entry.model,
name: entry.label,
reasoning: entry.session?.reasoning ?? false,
})),
};
});
}
backend/src/routes/pi-management.ts
+1 -9
View File
@@ -11,13 +11,6 @@ export function piManagementRoutes(
deps: { service: PiManagementService },
): void {
app.get("/pi-management/status", async (request, reply) => run(request, reply, deps, () => deps.service.status()));
app.get("/pi-management/options", async (request, reply) => run(request, reply, deps, () => deps.service.options()));
app.put("/pi-management/config", async (request, reply) => run(
request,
reply,
deps,
() => deps.service.configure((request.body ?? {}) as Record<string, unknown>),
));
app.post("/pi-management/test", async (request, reply) => run(request, reply, deps, () => deps.service.test()));
app.get("/pi-management/logs", async (request, reply) => run(request, reply, deps, () => deps.service.logs()));
}
@@ -38,8 +31,7 @@ async function run<T>(
return await action();
} catch (error) {
if (error instanceof PiManagementError) {
const statusCode = error.code === "pi_management_invalid_config" ? 400 : 503;
return reply.code(statusCode).send({ code: error.code, error: error.message });
return reply.code(503).send({ code: error.code, error: error.message });
}
return reply.code(503).send({ code: "pi_management_unavailable", error: "Pi management is unavailable" });
}
backend/src/routes/sessions.ts
+18 -9
View File
@@ -6,12 +6,13 @@ import type { Settings } from "../settings/settings-store.js";
import { getPrincipal } from "../auth/auth.js";
import type { PrincipalContext } from "../auth/principal.js";
import type { ReadinessManager } from "../runtime/readiness-manager.js";
import type { ListModelsFn } from "./meta.js";
import type { ListModelsFn } from "../pi/list-models.js";
import type { WorkspaceRegistry } from "../workspaces/registry.js";
import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js";
import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js";
import { splitCanonicalModelId, type RuntimeModelCatalog } from "../models/runtime-model-catalog.js";
const BOOTSTRAP_FAILURE_MESSAGE =
"Session startup failed. Check configuration and connectivity, then Resume the session.";
@@ -43,6 +44,7 @@ export function sessionRoutes(
maintenanceBarrier: MaintenanceBarrier;
/** Optional only for narrow route-test stubs and installations without a Catalog database. */
effectiveRelationships?: EffectiveRelationshipSnapshotProvider;
modelCatalog: RuntimeModelCatalog;
},
) {
const lifecycleTails = new Map<string, Promise<void>>();
@@ -358,7 +360,6 @@ export function sessionRoutes(
let workspaceId: string | undefined;
let workspaceRevision: string | undefined;
let workspaceDescriptor: WorkspaceDescriptor | undefined;
let allowedModels: readonly string[] | undefined;
if (requestedWorkspaceId) {
try {
const registry = d.workspaceRegistry as Partial<WorkspaceRegistry>;
@@ -378,7 +379,6 @@ export function sessionRoutes(
workspaceId = resolved.revision.id;
workspaceRevision = resolved.revision.commit;
workspaceDescriptor = resolved.workspace;
allowedModels = resolved.workspace.llm_policy.allowed;
} catch {
return reply.code(409).send({
error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE,
@@ -386,12 +386,17 @@ export function sessionRoutes(
});
}
}
const provider = b.provider ?? s.provider;
const model = b.model ?? s.model;
const requestedCanonical = b.provider && b.model ? `${b.provider}/${b.model}` : undefined;
let selectedCanonical = requestedCanonical ?? d.modelCatalog.defaultSession;
let modelWarning: string | undefined;
if (selectedCanonical && d.modelCatalog.defaultSession && !d.modelCatalog.hasSession(selectedCanonical)) {
selectedCanonical = d.modelCatalog.defaultSession;
modelWarning = `Configured model ${requestedCanonical ?? "selection"} is unavailable; using ${selectedCanonical}.`;
}
const selected = selectedCanonical ? splitCanonicalModelId(selectedCanonical) : undefined;
const provider = selected?.provider ?? b.provider;
const model = selected?.model ?? b.model;
const thinking = b.thinking ?? s.thinking;
if (allowedModels && provider && model && !allowedModels.includes(`${provider}/${model}`)) {
return reply.code(400).send({ error: "Selected model is not allowed by this workspace." });
}
// A persisted session is resumable without keeping Pi alive. New work replaces every
// runtime owned by this principal, while runtimes belonging to other users remain intact.
// Optional chaining preserves the deliberately narrow manager stubs used by route tests.
@@ -490,7 +495,7 @@ export function sessionRoutes(
),
() => d.mgr.start(id, rt, runtimeOptions),
);
return { id };
return { id, ...(modelWarning ? { warning: modelWarning } : {}) };
} finally {
if (revisionLease && !manifestPersisted) {
await revisionLease.abort().catch((error: unknown) => {
@@ -598,6 +603,10 @@ export function sessionRoutes(
provider?: string; model?: string; thinking?: string;
workspace_id?: string; workspace_revision?: string;
};
const savedCanonical = saved.provider && saved.model ? `${saved.provider}/${saved.model}` : "";
if (d.modelCatalog.defaultSession && (!savedCanonical || !d.modelCatalog.hasSession(savedCanonical))) {
return reply.code(503).send({ error: MODEL_UNAVAILABLE_MESSAGE, code: "model_unavailable" });
}
let workspaceConfigPath: string;
let workspaceDescriptor: WorkspaceDescriptor | undefined;
try {
backend/src/routes/settings.ts
+16 -22
View File
@@ -1,17 +1,27 @@
import type { FastifyInstance } from "fastify";
import type { AppConfig } from "../config.js";
import type { Settings } from "../settings/settings-store.js";
import { listWorkspaces, type ListModelsFn } from "./meta.js";
import { listWorkspaces } from "./meta.js";
import type { PrincipalContext } from "../auth/principal.js";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import {
splitCanonicalModelId,
type RuntimeModelCatalog,
} from "../models/runtime-model-catalog.js";
/** Merge stored settings over env/first-workspace defaults. */
export function effectiveSettings(cfg: AppConfig, stored: Settings): Settings {
/** Merge only workspace and runtime-thinking preferences; model defaults belong to modelCatalog. */
export function effectiveSettings(
cfg: AppConfig,
stored: Settings,
modelCatalog?: RuntimeModelCatalog,
): Settings {
const workspaces = listWorkspaces(cfg.harnessDir);
const selected = modelCatalog?.defaultSession
? splitCanonicalModelId(modelCatalog.defaultSession)
: undefined;
return {
workspace: stored.workspace ?? workspaces[0]?.name,
provider: cfg.defaults.provider ?? stored.provider,
model: cfg.defaults.model ?? stored.model,
...(selected ?? {}),
thinking: cfg.defaults.thinking ?? stored.thinking,
};
}
@@ -19,7 +29,7 @@ export function effectiveSettings(cfg: AppConfig, stored: Settings): Settings {
export function settingsRoutes(
app: FastifyInstance,
deps: {
cfg: AppConfig; listModels: ListModelsFn;
cfg: AppConfig;
getSettings: (principal: PrincipalContext) => Promise<Settings>;
},
): void {
@@ -37,22 +47,6 @@ export function settingsRoutes(
const principal = requirePermission(req, reply, "settings.manage");
if (!isPrincipalContext(principal)) return principal;
const b = (req.body ?? {}) as Settings;
if (b.model) {
let available: { provider: string; id: string }[] = [];
try {
available = await deps.listModels();
} catch {
available = [];
}
// Only validate when Pi gave us a non-empty list; otherwise allow (degraded).
if (available.length > 0 && !available.some(
(candidate) => candidate.provider === b.provider && candidate.id === b.model,
)) {
return reply.code(400).send({
error: `Unknown model: ${b.provider ?? "unknown"}/${b.model}`,
});
}
}
try {
// Retain this endpoint as a validating compatibility surface for older clients, but do
// not write anonymous users' choices to shared server storage.
backend/src/settings/settings-store.ts
+8 -1
View File
@@ -13,6 +13,7 @@ import type { AppConfig } from "../config.js";
export interface Settings {
workspace?: string;
/** Legacy input fields are ignored when loading/evaluating installation settings. */
provider?: string;
model?: string;
thinking?: string;
@@ -37,7 +38,13 @@ export function loadSettings(cfg: AppConfig): Settings {
try {
const raw = readFileSync(cfg.settingsFile, "utf8");
const parsed = JSON.parse(raw);
if (parsed && typeof parsed === "object") return parsed as Settings;
if (parsed && typeof parsed === "object") {
const value = parsed as Record<string, unknown>;
return {
...(typeof value.workspace === "string" ? { workspace: value.workspace } : {}),
...(typeof value.thinking === "string" ? { thinking: value.thinking } : {}),
};
}
return {};
} catch {
return {};
backend/src/tht/tht-runner.ts
+5 -1
View File
@@ -598,7 +598,11 @@ export class ThtRunner {
} catch {
return { ok: false, code: "workspace_not_activatable" };
}
const collection = descriptor.semantic_index.vector_store;
const collection = {
collection: descriptor.workspace.id,
dimensions: this.cfg.semanticRuntime.internalEmbeddingDimensions,
distance: "cosine" as const,
};
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutSec) * 1000);
try {
backend/src/workspace-maintenance.ts
+1
View File
@@ -238,6 +238,7 @@ function createProductionService(): WorkspacePreprocessingService {
});
return new WorkspacePreprocessingService({
dataRoot: config.dataRoot ?? "/data",
embeddingDimensions: config.internalEmbeddingDimensions,
httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "")
.split(",").map((value) => value.trim()).filter((value) => value.length > 0),
acquireActiveRuntime: async (workspaceId) => {
backend/src/workspaces/bindings.ts
+4 -4
View File
@@ -59,12 +59,12 @@ function safeSecretFilePath(path: string, secretRoots: readonly string[]): strin
function requireSupportedDescriptor(workspace: unknown): void {
if (typeof workspace !== "object" || workspace === null) {
throw new Error("Workspace bindings support only workspace schema version 3");
throw new Error("Workspace bindings support only workspace schema version 4");
}
const metadata = Reflect.get(workspace, "workspace");
if (typeof metadata !== "object" || metadata === null
|| Reflect.get(metadata, "schema_version") !== 3) {
throw new Error("Workspace bindings support only workspace schema version 3");
|| Reflect.get(metadata, "schema_version") !== 4) {
throw new Error("Workspace bindings support only workspace schema version 4");
}
}
@@ -155,7 +155,7 @@ export function resolveEvidenceBinding(
return { values, missing };
}
/** Resolve the complete schema-v3 runtime binding set. */
/** Resolve the complete schema-v4 runtime binding set. */
export function resolveRuntimeBindings(
workspace: WorkspaceDescriptor,
env: NodeJS.ProcessEnv,
backend/src/workspaces/contracts.ts
+3 -3
View File
@@ -137,12 +137,12 @@ function evidenceVariables(
function requireSupportedDescriptor(workspace: unknown): void {
if (typeof workspace !== "object" || workspace === null) {
throw new Error("Installation contract supports only workspace schema version 3");
throw new Error("Installation contract supports only workspace schema version 4");
}
const metadata = Reflect.get(workspace, "workspace");
if (typeof metadata !== "object" || metadata === null
|| Reflect.get(metadata, "schema_version") !== 3) {
throw new Error("Installation contract supports only workspace schema version 3");
|| Reflect.get(metadata, "schema_version") !== 4) {
throw new Error("Installation contract supports only workspace schema version 4");
}
}
backend/src/workspaces/diagnostics.ts
+11 -9
View File
@@ -406,12 +406,12 @@ function numericBinding(binding: Record<string, string>, name: string): number |
function requireSupportedDescriptor(workspace: unknown): void {
if (typeof workspace !== "object" || workspace === null) {
throw new Error("Workspace diagnoser supports only workspace schema version 3");
throw new Error("Workspace diagnoser supports only workspace schema version 4");
}
const metadata = Reflect.get(workspace, "workspace");
if (typeof metadata !== "object" || metadata === null
|| Reflect.get(metadata, "schema_version") !== 3) {
throw new Error("Workspace diagnoser supports only workspace schema version 3");
|| Reflect.get(metadata, "schema_version") !== 4) {
throw new Error("Workspace diagnoser supports only workspace schema version 4");
}
}
@@ -506,11 +506,15 @@ async function diagnoseValidatedWorkspace(
try {
const vector = await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({
baseUrl: semanticRuntime.internalQdrantUrl,
collection: descriptor.semantic_index.vector_store.collection,
collection: descriptor.workspace.id,
timeoutMs,
signal,
}));
const expected = descriptor.semantic_index.vector_store;
const expected = {
collection: descriptor.workspace.id,
dimensions: semanticRuntime.internalEmbeddingDimensions,
distance: "cosine",
};
if (vector.collection !== expected.collection
|| vector.dimensions !== expected.dimensions
|| vector.distance !== expected.distance) {
@@ -529,10 +533,8 @@ async function diagnoseValidatedWorkspace(
timeoutMs,
signal,
}));
if (semanticRuntime.internalEmbeddingModel !== descriptor.semantic_index.embedding.model
|| semanticRuntime.internalEmbeddingDimensions !== descriptor.semantic_index.embedding.dimensions
|| !embedding.available
|| embedding.dimensions !== descriptor.semantic_index.embedding.dimensions) {
if (!embedding.available
|| embedding.dimensions !== semanticRuntime.internalEmbeddingDimensions) {
diagnostics.push(diagnosticError("semantic_index_incompatible"));
activatable = false;
}
backend/src/workspaces/effective-config.ts
+6 -3
View File
@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
import { normalize } from "node:path";
export interface CanonicalEffectiveConfig {
schemaVersion: 1;
schemaVersion: 2;
dwh: CanonicalDwhConfig;
vector: CanonicalVectorConfig;
embedding: CanonicalEmbeddingConfig;
@@ -27,6 +27,7 @@ export interface CanonicalVectorConfig {
}
export interface CanonicalEmbeddingConfig {
id: string;
model: string;
dimensions: number;
}
@@ -137,8 +138,10 @@ function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbed
if (!embeddings) {
throw new TypeError("effective config is missing embedding resources");
}
const model = requireString(embeddings, "model");
return {
model: requireString(embeddings, "model"),
id: `ollama/${model}`,
model,
dimensions: requireNumber(embeddings, "dimensions"),
};
}
@@ -166,7 +169,7 @@ export function buildCanonicalEffectiveConfig(renderedConfig: unknown): Canonica
throw new TypeError("effective config requires a rendered configuration object");
}
return {
schemaVersion: 1,
schemaVersion: 2,
dwh: buildDwhConfig(rendered),
vector: buildVectorConfig(rendered),
embedding: buildEmbeddingConfig(rendered),
backend/src/workspaces/preprocessing-service.ts
+7 -4
View File
@@ -77,6 +77,7 @@ export interface WorkspacePreprocessingServiceDeps {
{ ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
>;
httpPrivateHostAllowlist?: readonly string[];
embeddingDimensions?: number;
}
interface RunScope {
@@ -118,7 +119,7 @@ export class WorkspacePreprocessingService {
async vectorInspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
const collection = runtime.workspace.semantic_index.vector_store.collection;
const collection = runtime.workspace.workspace.id;
const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" });
if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] });
const body = await res.json() as any;
@@ -132,7 +133,7 @@ export class WorkspacePreprocessingService {
async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise<WorkspaceOperationResult> {
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
const collection = runtime.workspace.semantic_index.vector_store.collection;
const collection = runtime.workspace.workspace.id;
if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) {
return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] });
}
@@ -143,8 +144,8 @@ export class WorkspacePreprocessingService {
const recreated = await reconcileCollection({
baseUrl: runtime.configLease.semanticQdrantUrl,
collection,
dimensions: runtime.workspace.semantic_index.vector_store.dimensions,
distance: runtime.workspace.semantic_index.vector_store.distance,
dimensions: this.deps.embeddingDimensions ?? 1024,
distance: "cosine",
mode: "self_heal",
});
if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] });
@@ -400,6 +401,8 @@ export class WorkspacePreprocessingService {
catalogBlob: runtime.catalogBlob,
configDigest: runtime.configLease.configDigest,
bindingDigest: runtime.configLease.bindingDigest,
embeddingId: runtime.configLease.effectiveConfig.embedding.id,
embeddingDimensions: runtime.configLease.effectiveConfig.embedding.dimensions,
});
return { runtime, state, job };
}
backend/src/workspaces/preprocessing-state.ts
+13 -3
View File
@@ -44,11 +44,13 @@ export interface BeginPreprocessingJobOptions {
catalogBlob: string;
configDigest: string;
bindingDigest: string;
embeddingId: string;
embeddingDimensions: number;
runId?: string;
}
export interface PreprocessingJobState {
schemaVersion: 1;
schemaVersion: 2;
runId: string;
operation: string;
workspaceId: string;
@@ -57,6 +59,8 @@ export interface PreprocessingJobState {
catalogBlob: string;
configDigest: string;
bindingDigest: string;
embeddingId: string;
embeddingDimensions: number;
completedStages: string[];
childRuns: Record<string, string>;
status: "active" | "succeeded" | "blocked" | "failed";
@@ -146,7 +150,7 @@ function decodeJob(value: unknown): PreprocessingJobState {
}
const record = value as Record<string, unknown>;
if (
record.schemaVersion !== 1
record.schemaVersion !== 2
|| typeof record.runId !== "string"
|| typeof record.operation !== "string"
|| typeof record.workspaceId !== "string"
@@ -155,6 +159,8 @@ function decodeJob(value: unknown): PreprocessingJobState {
|| typeof record.catalogBlob !== "string"
|| typeof record.configDigest !== "string"
|| typeof record.bindingDigest !== "string"
|| typeof record.embeddingId !== "string"
|| typeof record.embeddingDimensions !== "number"
|| !Array.isArray(record.completedStages)
|| typeof record.childRuns !== "object" || record.childRuns === null || Array.isArray(record.childRuns)
|| !["active", "succeeded", "blocked", "failed"].includes(String(record.status))
@@ -275,6 +281,8 @@ export class PreprocessingStateStore {
|| existing.catalogBlob !== options.catalogBlob
|| existing.configDigest !== options.configDigest
|| existing.bindingDigest !== options.bindingDigest
|| existing.embeddingId !== options.embeddingId
|| existing.embeddingDimensions !== options.embeddingDimensions
) {
throw new PreprocessingStateError(
"preprocessing_resume_mismatch",
@@ -295,7 +303,7 @@ export class PreprocessingStateStore {
}
}
const job: PreprocessingJobState = {
schemaVersion: 1,
schemaVersion: 2,
runId,
operation: options.operation,
workspaceId: this.options.workspaceId,
@@ -304,6 +312,8 @@ export class PreprocessingStateStore {
catalogBlob: options.catalogBlob,
configDigest: options.configDigest,
bindingDigest: options.bindingDigest,
embeddingId: options.embeddingId,
embeddingDimensions: options.embeddingDimensions,
completedStages: [],
childRuns: {},
status: "active",
backend/src/workspaces/registry.ts
+1 -1
View File
@@ -560,7 +560,7 @@ export class WorkspaceRegistry {
});
}
}
const collection = workspace.semantic_index.vector_store.collection;
const collection = workspace.workspace.id;
const owner = collectionOwners.get(collection);
if (owner !== undefined) {
throw new Error(`duplicate qdrant collection ownership: ${collection} (${owner}, ${id})`);
backend/src/workspaces/runtime-renderer.ts
+20 -6
View File
@@ -34,6 +34,7 @@ export interface RuntimeInstallationOverlay {
export interface SemanticRuntimeConfig {
internalQdrantUrl: string;
internalEmbeddingUrl: string;
internalEmbeddingId?: string;
internalEmbeddingModel: string;
internalEmbeddingDimensions: number;
}
@@ -41,6 +42,7 @@ export interface SemanticRuntimeConfig {
export const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
internalQdrantUrl: "http://qdrant:6333",
internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
internalEmbeddingModel: "qwen3-embedding:0.6b",
internalEmbeddingDimensions: 1024,
};
@@ -202,16 +204,16 @@ function placeholderConnection(identity: { database: string; schema: string }):
function requireSupportedDescriptor(workspace: unknown): void {
if (typeof workspace !== "object" || workspace === null) {
throw new Error("Runtime renderer supports only workspace schema version 3");
throw new Error("Runtime renderer supports only workspace schema version 4");
}
const metadata = Reflect.get(workspace, "workspace");
if (typeof metadata !== "object" || metadata === null
|| Reflect.get(metadata, "schema_version") !== 3) {
throw new Error("Runtime renderer supports only workspace schema version 3");
|| Reflect.get(metadata, "schema_version") !== 4) {
throw new Error("Runtime renderer supports only workspace schema version 4");
}
}
/** Render the schema-v3 compatibility fields consumed by the current Python harness. */
/** Render the schema-v4 compatibility fields consumed by the current Python harness. */
export function renderRuntimeConfig(
workspace: WorkspaceDescriptor,
bindings: RuntimeBindings,
@@ -263,12 +265,24 @@ export function renderRuntimeConfig(
...(installation.profile === undefined ? {} : { profile: installation.profile }),
language: descriptor.workspace.language,
database,
semantic_index: descriptor.semantic_index,
semantic_index: {
vector_store: {
engine: "qdrant",
collection: descriptor.workspace.id,
dimensions: semanticRuntime.internalEmbeddingDimensions,
distance: "cosine",
},
embedding: {
provider: "ollama_internal",
model: semanticRuntime.internalEmbeddingModel,
dimensions: semanticRuntime.internalEmbeddingDimensions,
},
},
resources: {
vector: {
engine: "qdrant",
base_url: semanticRuntime.internalQdrantUrl,
collection: descriptor.semantic_index.vector_store.collection,
collection: descriptor.workspace.id,
},
embeddings: {
provider: "ollama_internal",
backend/src/workspaces/schema.ts
+29 -67
View File
@@ -35,7 +35,7 @@ export interface CanonicalDiagnostics {
}
interface WorkspaceMetadata {
schema_version: 3;
schema_version: 4;
id: string;
name: string;
description?: string;
@@ -51,31 +51,12 @@ interface WorkspaceDwh {
supported_transports: DwhTransport[];
}
interface WorkspaceBase<TVectorStore> {
interface WorkspaceBase {
workspace: WorkspaceMetadata;
dwh: WorkspaceDwh;
semantic_index: {
vector_store: TVectorStore;
embedding: {
provider: "ollama_internal";
model: "qwen3-embedding:0.6b";
dimensions: 1024;
};
};
llm_policy: {
default?: `${string}/${string}`;
allowed: `${string}/${string}`[];
};
diagnostics?: Pick<CanonicalDiagnostics, "dwh_rest">;
}
interface QdrantVectorStore {
engine: "qdrant";
collection: string;
dimensions: 1024;
distance: "cosine";
}
export interface EvidencePolicy {
max_chunk_chars: number;
retain_published_generations: number;
@@ -120,12 +101,12 @@ export interface WorkspaceEvidence {
policy: EvidencePolicy;
}
export interface WorkspaceV3 extends WorkspaceBase<QdrantVectorStore> {
export interface WorkspaceV4 extends WorkspaceBase {
evidence?: WorkspaceEvidence;
}
export type CanonicalWorkspace = WorkspaceV3;
export type WorkspaceDescriptor = WorkspaceV3;
export type CanonicalWorkspace = WorkspaceV4;
export type WorkspaceDescriptor = WorkspaceV4;
const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, {
message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$",
@@ -135,9 +116,6 @@ const identifier = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, {
});
const port = z.number().int().min(1).max(65_535);
const timeoutMs = z.number().int().positive();
const modelReference = z.string().regex(/^[^/\s]+\/[^/\s]+$/, {
message: "model must use provider/model syntax",
});
function isOriginRelativeDiagnosticPath(value: string): boolean {
return /^\/(?!\/)[^\\\u0000-\u001F\u007F?#]*$/.test(value) && !/%5c/i.test(value);
@@ -166,21 +144,6 @@ const dwhSchema = z.object({
timeout_ms: timeoutMs.optional(),
supported_transports: z.array(z.enum(DWH_TRANSPORTS)).min(1),
}).strict();
const internalEmbeddingSchema = z.object({
provider: z.literal("ollama_internal"),
model: z.literal("qwen3-embedding:0.6b"),
dimensions: z.literal(1024),
}).strict();
const qdrantVectorStoreSchema = z.object({
engine: z.literal("qdrant"),
collection: workspaceId,
dimensions: z.literal(1024),
distance: z.literal("cosine"),
}).strict();
const llmPolicySchema = z.object({
default: modelReference.optional(),
allowed: z.array(modelReference).min(1),
}).strict();
const positiveSafeInteger = z.number().int().safe().positive();
const nonnegativeSafeInteger = z.number().int().safe().nonnegative();
@@ -366,7 +329,6 @@ function unique<T>(values: readonly T[], context: z.RefinementCtx, path: Propert
function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
unique(workspace.llm_policy.allowed, context, ["llm_policy", "allowed"]);
if (workspace.evidence?.source.type === "filesystem") {
const expected = `${workspace.workspace.id}/evidence`;
@@ -379,20 +341,6 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
}
}
if (workspace.semantic_index.vector_store.dimensions !== workspace.semantic_index.embedding.dimensions) {
context.addIssue({
code: "custom",
path: ["semantic_index", "embedding", "dimensions"],
message: "embedding dimensions must match vector store dimensions",
});
}
if (workspace.llm_policy.default && !workspace.llm_policy.allowed.includes(workspace.llm_policy.default)) {
context.addIssue({
code: "custom",
path: ["llm_policy", "default"],
message: "LLM default must be included in the allowlist",
});
}
if (workspace.diagnostics?.dwh_rest && !workspace.dwh.supported_transports.includes("rest_api")) {
context.addIssue({
code: "custom",
@@ -402,23 +350,18 @@ function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
}
}
const WorkspaceV3Schema = z.object({
const WorkspaceV4Schema = z.object({
dwh: dwhSchema,
llm_policy: llmPolicySchema,
evidence: workspaceEvidenceSchema.optional(),
diagnostics: z.object({
dwh_rest: dwhRestDiagnostic.optional(),
}).strict().optional(),
workspace: z.object({
schema_version: z.literal(3), id: workspaceId, name: z.string().trim().min(1),
schema_version: z.literal(4), id: workspaceId, name: z.string().trim().min(1),
description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]),
}).strict(),
semantic_index: z.object({
vector_store: qdrantVectorStoreSchema,
embedding: internalEmbeddingSchema,
}).strict(),
}).strict().superRefine(workspaceInvariants);
const WorkspaceDescriptorSchema = WorkspaceV3Schema;
const WorkspaceDescriptorSchema = WorkspaceV4Schema;
export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
const documents = parseAllDocuments(source, { uniqueKeys: true });
@@ -431,6 +374,25 @@ export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
return validateWorkspaceDescriptor(document.toJSON());
}
/** Deterministically removes the two installation-owned v3 blocks without altering workspace data. */
export function migrateWorkspaceV3Yaml(source: string): string {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
const document = documents[0];
if (document.errors.length > 0 || document.warnings.length > 0) {
throw new Error("Invalid workspace YAML");
}
const value = document.toJSON() as Record<string, unknown>;
const metadata = value.workspace as Record<string, unknown> | undefined;
if (!metadata || metadata.schema_version !== 3) {
throw new Error("Workspace migration requires schema version 3");
}
metadata.schema_version = 4;
delete value.semantic_index;
delete value.llm_policy;
return serializeWorkspaceYaml(validateWorkspaceDescriptor(value));
}
export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescriptor {
return WorkspaceDescriptorSchema.parse(workspace) as WorkspaceDescriptor;
}
@@ -439,11 +401,11 @@ export function isCanonicalWorkspace(workspace: unknown): workspace is Canonical
return WorkspaceDescriptorSchema.safeParse(workspace).success;
}
export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV3 {
export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV4 {
return WorkspaceDescriptorSchema.safeParse(workspace).success;
}
export function validateOperationalWorkspace(workspace: unknown): WorkspaceV3 {
export function validateOperationalWorkspace(workspace: unknown): WorkspaceV4 {
return validateWorkspaceDescriptor(workspace);
}
backend/src/workspaces/types.ts
+1 -1
View File
@@ -19,4 +19,4 @@ export type WorkspaceErrorCode =
| "workspace_stale" | "git_unavailable" | "git_auth_failed" | "git_non_fast_forward"
| "connector_unavailable" | "semantic_index_incompatible";
export type { WorkspaceV3 } from "./schema.js";
export type { WorkspaceV4 } from "./schema.js";