mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: configure workspace runtime secrets through API

Browse Source
This commit is contained in:
marcopan
2026-08-14 17:30:35 +02:00
parent 2114c94704
commit 87cefd120c
7 changed files with 325 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/app.ts
+26 -6
View File
@@ -21,8 +21,10 @@ import { WorkspaceRegistry } from "./workspaces/registry.js";
import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js";
import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js";
import { piManagementRoutes } from "./routes/pi-management.js";
import { resolveRuntimeBindings, supportsSessionRuntime } from "./workspaces/bindings.js";
import { supportsSessionRuntime } from "./workspaces/bindings.js";
import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js";
import type { WorkspaceDescriptor } from "./workspaces/schema.js";
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
export interface BuildAppDeps {
thtRunner?: ThtRunner;
@@ -34,6 +36,7 @@ export interface BuildAppDeps {
hub?: SseHub;
workspaceRegistry?: WorkspaceRegistry;
workspaceDiagnoser?: WorkspaceDiagnoser;
workspaceSecretStore?: WorkspaceSecretStore;
workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
maintenanceBarrier?: MaintenanceBarrier;
piManagement?: PiManagementService;
@@ -41,6 +44,11 @@ export interface BuildAppDeps {
export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstance {
const app = Fastify({ logger: { level: "warn" }, disableRequestLogging: true });
const workspaceSecretStore = deps?.workspaceSecretStore ?? new WorkspaceSecretStore({
root: config.workspaceSecretStoreRoot,
runtimeRoot: config.workspaceSecretRuntimeRoot,
installationId: config.workspaceRegistry.installationId,
});
// Allow any origin in dev/e2e; tighten in production via config if needed.
app.register(cors, {
@@ -58,6 +66,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
secretRoots: config.workspaceRegistry.secretRoots,
secretsFile: config.secretsFile,
secretFiles: config.secretFiles,
workspaceSecretStore,
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
@@ -75,13 +84,19 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
internalEmbeddingModel: config.internalEmbeddingModel,
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
});
const workspaceRuntimeSupport = deps?.workspaceRuntimeSupport ?? ((workspace: WorkspaceDescriptor) => (
supportsSessionRuntime(resolveRuntimeBindings(
const workspaceRuntimeSupport = deps?.workspaceRuntimeSupport ?? ((workspace: WorkspaceDescriptor) => {
const lease = resolveRuntimeBindingsWithWorkspaceSecrets(
workspace,
process.env,
config.workspaceRegistry.secretRoots,
))
));
workspaceSecretStore,
);
try {
return supportsSessionRuntime(lease.bindings);
} finally {
lease.release();
}
});
const readiness = deps?.readiness ?? new ReadinessManager(
tht as ThtRunner,
Math.round(config.ollamaEnsureTimeoutMs / 1000),
@@ -180,7 +195,12 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
});
sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
metaRoutes(app, { harnessDir: config.harnessDir, listModels });
workspaceRoutes(app, { registry: workspaceRegistry, config: config.workspaceRegistry, diagnose: workspaceDiagnoser });
workspaceRoutes(app, {
registry: workspaceRegistry,
config: config.workspaceRegistry,
diagnose: workspaceDiagnoser,
secretStore: workspaceSecretStore,
});
settingsRoutes(app, { cfg: config, listModels, getSettings });
piManagementRoutes(app, { config, service: piManagement });
backend/src/config.ts
+12
View File
@@ -30,6 +30,8 @@ export interface AppConfig {
legacyWorkspaceMode: boolean;
workspaceDiagnosticTimeoutMs: number;
workspaceRegistry: WorkspaceRegistryConfig;
workspaceSecretStoreRoot: string;
workspaceSecretRuntimeRoot: string;
internalQdrantUrl: string;
internalEmbeddingUrl: string;
internalEmbeddingModel: string;
@@ -248,6 +250,14 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
maxEvidencePathBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_EVIDENCE_PATH_BYTES, 4096),
maxEvidenceManifestBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_EVIDENCE_MANIFEST_BYTES, 1024 * 1024),
};
const workspaceSecretStoreRoot = absoluteRegistryPath(
env.THT_WORKSPACE_SECRET_STORE_ROOT ?? path.join(path.dirname(registryRoot), "workspace-secrets"),
"secret store root",
);
const workspaceSecretRuntimeRoot = absoluteRegistryPath(
env.THT_WORKSPACE_SECRET_RUNTIME_ROOT ?? "/tmp/thothii-workspace-secrets",
"secret runtime root",
);
const settingsFile = env.SETTINGS_FILE ?? "data/settings.json";
const internalQdrantUrl = internalServiceUrl(
env.THT_INTERNAL_QDRANT_URL,
@@ -284,6 +294,8 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
legacyWorkspaceMode: legacyWorkspaceMode === "local",
workspaceDiagnosticTimeoutMs: diagnosticTimeout(env.THT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS),
workspaceRegistry,
workspaceSecretStoreRoot,
workspaceSecretRuntimeRoot,
internalQdrantUrl,
internalEmbeddingUrl,
internalEmbeddingModel: env.THT_INTERNAL_EMBEDDING_MODEL ?? "qwen3-embedding:0.6b",
backend/src/routes/workspaces.ts
+104 -12
View File
@@ -3,8 +3,12 @@ import { z } from "zod";
import type { WorkspaceRegistryConfig } from "../workspaces/types.js";
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
import type { WorkspaceRegistry } from "../workspaces/registry.js";
import { resolveRuntimeBindings } from "../workspaces/bindings.js";
import { buildInstallationContract } from "../workspaces/contracts.js";
import {
discoverWorkspaceSecretRequirements,
resolveRuntimeBindingsWithWorkspaceSecrets,
} from "../workspaces/secret-requirements.js";
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
import {
validateOperationalWorkspace,
validateWorkspaceDescriptor,
@@ -24,10 +28,17 @@ interface WorkspaceRoutesDeps {
registry: WorkspaceRegistry;
config: WorkspaceRegistryConfig;
diagnose: WorkspaceDiagnoser;
secretStore: WorkspaceSecretStore;
}
const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const workspacePayload = z.object({ workspace: z.unknown() }).strict();
const secretRequirementId = z.string().regex(/^[a-z0-9][a-z0-9._-]{1,127}$/);
const secretValuesPayload = z.object({
values: z.record(secretRequirementId, z.string()).refine(
(values) => Object.keys(values).length > 0 && Object.keys(values).length <= 16,
),
}).strict();
const SAFE_MESSAGES = {
workspace_invalid: "Workspace request is invalid.",
@@ -57,6 +68,29 @@ function errorReply(reply: FastifyReply, error: unknown) {
}
export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps): void {
const runtimeConfiguration = async (id: string) => {
const { workspace, revision } = await deps.registry.read(id);
const operational = validateOperationalWorkspace(workspace);
const requirements = discoverWorkspaceSecretRequirements(operational, process.env)
.map((requirement) => ({
id: requirement.id,
connector: requirement.connector,
label: requirement.label,
description: requirement.description,
input: requirement.input,
required: requirement.required,
configured: deps.secretStore.has(id, requirement.id),
}));
return {
workspaceId: id,
revision,
configurationState: requirements.some(({ required, configured }) => required && !configured)
? "configuration_required" as const
: "ready" as const,
requirements,
};
};
app.get("/workspace-registry/status", async (_request, reply) => {
try {
return await deps.registry.bootstrap();
@@ -76,15 +110,18 @@ export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps)
app.get("/workspaces", async (_request, reply) => {
try {
const records = await deps.registry.listCatalog();
return records.map((record) => ({
id: record.id,
// Retain the metadata endpoint's selector field while adding catalog metadata.
name: record.id,
file: `${record.id}/workspace.yaml`,
displayName: record.name,
description: record.description,
configurationState: record.configurationState,
...(record.revision ? { revision: record.revision } : {}),
return await Promise.all(records.map(async (record) => {
const configuration = await runtimeConfiguration(record.id);
return {
id: record.id,
// Retain the metadata endpoint's selector field while adding catalog metadata.
name: record.id,
file: `${record.id}/workspace.yaml`,
displayName: record.name,
description: record.description,
configurationState: configuration.configurationState,
...(record.revision ? { revision: record.revision } : {}),
};
}));
} catch (error) {
return errorReply(reply, error);
@@ -110,6 +147,52 @@ export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps)
}
});
app.get("/workspaces/:id/runtime-configuration", async (request, reply) => {
try {
const { id } = z.object({ id: workspaceId }).parse(request.params);
return await runtimeConfiguration(id);
} catch (error) {
return errorReply(reply, error);
}
});
app.put("/workspaces/:id/secrets", async (request, reply) => {
try {
const { id } = z.object({ id: workspaceId }).parse(request.params);
const { values } = secretValuesPayload.parse(request.body);
const { workspace } = await deps.registry.read(id);
const declared = new Set(
discoverWorkspaceSecretRequirements(validateOperationalWorkspace(workspace), process.env)
.map(({ id: requirementId }) => requirementId),
);
if (Object.keys(values).some((requirementId) => !declared.has(requirementId))) {
throw new Error("undeclared workspace secret");
}
deps.secretStore.putMany(id, values);
return await runtimeConfiguration(id);
} catch (error) {
return errorReply(reply, error);
}
});
app.delete("/workspaces/:id/secrets/:requirementId", async (request, reply) => {
try {
const { id, requirementId } = z.object({
id: workspaceId,
requirementId: secretRequirementId,
}).parse(request.params);
const { workspace } = await deps.registry.read(id);
const declared = discoverWorkspaceSecretRequirements(
validateOperationalWorkspace(workspace), process.env,
).some(({ id: candidate }) => candidate === requirementId);
if (!declared) throw new Error("undeclared workspace secret");
deps.secretStore.forget(id, requirementId);
return await runtimeConfiguration(id);
} catch (error) {
return errorReply(reply, error);
}
});
app.post("/workspaces/:id/test", async (request, reply) => {
try {
const { id } = z.object({ id: workspaceId }).parse(request.params);
@@ -122,8 +205,17 @@ export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps)
"workspace_not_activatable", "Workspace requires explicit migration",
);
}
const bindings = resolveRuntimeBindings(operational, process.env, deps.config.secretRoots);
return await deps.diagnose(operational, bindings, { writeProbe: false });
const lease = resolveRuntimeBindingsWithWorkspaceSecrets(
operational,
process.env,
deps.config.secretRoots,
deps.secretStore,
);
try {
return await deps.diagnose(operational, lease.bindings, { writeProbe: false });
} finally {
lease.release();
}
} catch (error) {
return errorReply(reply, error);
}
backend/src/tht/tht-runner.ts
+11 -1
View File
@@ -21,6 +21,7 @@ import {
type WorkspaceDescriptor,
} from "../workspaces/schema.js";
import { reconcileCollection } from "../workspaces/qdrant-collection.js";
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
export interface ThtConfig extends SecretBundleConfig {
thtBin: string;
@@ -33,6 +34,7 @@ export interface ThtConfig extends SecretBundleConfig {
qdrantRequest?: typeof fetch;
/** "self_heal" for session admission (create missing collections/indexes), default "require_existing". */
qdrantCollectionMode?: "self_heal" | "require_existing";
workspaceSecretStore?: WorkspaceSecretStore;
}
export interface RuntimeConfigLease {
@@ -221,8 +223,15 @@ export class ThtRunner {
})(),
secretRoots: this.cfg.secretRoots ?? [],
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
workspaceSecretStore: this.cfg.workspaceSecretStore,
});
const path = this.createRuntimeSnapshot(rendered.renderedConfig);
let path: string;
try {
path = this.createRuntimeSnapshot(rendered.renderedConfig);
} catch (error) {
rendered.releaseSecrets();
throw error;
}
let released = false;
return {
path,
@@ -232,6 +241,7 @@ export class ThtRunner {
if (released) return;
released = true;
this.cleanupRuntimeSnapshot(path);
rendered.releaseSecrets();
},
};
}
backend/src/workspaces/runtime-config-lease.ts
+43 -20
View File
@@ -27,6 +27,8 @@ import {
type CanonicalEffectiveConfig,
} from "./effective-config.js";
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./secret-requirements.js";
import type { WorkspaceSecretStore } from "./secret-store.js";
import { GitWorkspaceRepository } from "./git-repository.js";
import { WorkspaceRegistry } from "./registry.js";
import {
@@ -58,6 +60,7 @@ export interface RenderedWorkspaceRuntime {
bindingDigest: string;
renderedConfig: string;
semanticQdrantUrl: string;
releaseSecrets(): void;
}
export interface ActiveRenderedWorkspaceRuntime extends RenderedWorkspaceRuntime {
@@ -298,33 +301,49 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
}): RenderedWorkspaceRuntime {
const bindings = resolveRuntimeBindings(options.workspace, process.env, options.secretRoots);
const secretLease = options.workspaceSecretStore === undefined
? undefined
: resolveRuntimeBindingsWithWorkspaceSecrets(
options.workspace,
process.env,
options.secretRoots,
options.workspaceSecretStore,
);
const bindings = secretLease?.bindings
?? resolveRuntimeBindings(options.workspace, process.env, options.secretRoots);
const overlay = installationOverlay(options.harnessDir, options.configPath);
const context: RuntimeRenderContext = {
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
};
return {
workspace: options.workspace,
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
runtimePaths: runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
installationOverlay: overlay,
bindings,
bindingDigest: stableBindingDigest(bindings),
semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
renderedConfig: renderRuntimeConfig(
options.workspace,
try {
return {
workspace: options.workspace,
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
runtimePaths: runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
installationOverlay: overlay,
bindings,
runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
context,
overlay,
options.semanticRuntime,
),
};
bindingDigest: stableBindingDigest(bindings),
semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
releaseSecrets: () => secretLease?.release(),
renderedConfig: renderRuntimeConfig(
options.workspace,
bindings,
runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
context,
overlay,
options.semanticRuntime,
),
};
} catch (error) {
secretLease?.release();
throw error;
}
}
export function renderWorkspaceRuntimeFromSnapshotPath(options: {
@@ -334,6 +353,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
}): RenderedWorkspaceRuntime {
const snapshot = readSnapshotWorkspace(options.snapshotPath);
return renderWorkspaceRuntimeFromWorkspace({
@@ -346,6 +366,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: options.dataRoot,
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
workspaceSecretStore: options.workspaceSecretStore,
});
}
@@ -358,6 +379,7 @@ export async function renderActiveWorkspaceRuntime(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
}): Promise<ActiveRenderedWorkspaceRuntime> {
// The persisted active state may reference host-side snapshot paths (written by another
// process or installation). Read the active state directly and resolve the immutable snapshot
@@ -387,6 +409,7 @@ export async function renderActiveWorkspaceRuntime(options: {
dataRoot: options.dataRoot,
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
workspaceSecretStore: options.workspaceSecretStore,
});
return {
...rendered,
@@ -584,4 +607,4 @@ export async function publishDeterministicRuntimeConfigLease(options: {
inputFingerprint: inputFingerprintValue,
release: () => undefined,
};
}
}