feat: render private semantic service endpoints

This commit is contained in:
2026-08-08 17:17:02 +02:00
parent f7b9f3276b
commit bc8afe0205
10 changed files with 372 additions and 19 deletions
@@ -0,0 +1,92 @@
# Task 3 report — Remove external semantic bindings and render internal endpoints
Date: 2026-08-08
## Scope
Implemented backend-owned schema-v3 semantic runtime rendering so workspace descriptors and installation contracts remain free of external Qdrant/Ollama endpoints and credentials, while DWH bindings stay unchanged.
## RED evidence
Focused RED command:
`cd backend && npx vitest run test/workspaces-contracts.test.ts test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts test/config.test.ts`
Observed failures before implementation:
- `config.test.ts`
- missing `internalQdrantUrl`
- missing `internalEmbeddingUrl`
- `workspaces-bindings.test.ts`
- schema v3 semantic binding resolution threw unsupported errors
- `workspace-runtime-renderer.test.ts`
- schema v3 runtime rendering threw `Schema version 3 runtime rendering is unsupported until the internal semantic runtime is implemented`
## GREEN evidence
Focused GREEN command:
`cd backend && npx vitest run test/workspaces-contracts.test.ts test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts test/config.test.ts`
Result:
- 4 test files passed
- 36 tests passed
Typecheck:
`cd backend && npx tsc --noEmit -p .`
Result:
- passed
Hygiene:
- `git diff --check` passed
## Files changed
Listed-task files changed:
- `backend/src/config.ts`
- `backend/src/workspaces/bindings.ts`
- `backend/src/workspaces/runtime-renderer.ts`
- `backend/test/config.test.ts`
- `backend/test/workspace-runtime-renderer.test.ts`
- `backend/test/workspaces-bindings.test.ts`
- `backend/test/workspaces-contracts.test.ts`
Listed-task files inspected but not changed:
- `backend/src/workspaces/contracts.ts`
Unavoidable additional wiring changes:
- `backend/src/app.ts`
- `backend/src/tht/tht-runner.ts`
Reason: the new typed internal semantic runtime config had to flow from backend config into ephemeral harness config rendering at runtime.
## Behavior delivered
- schema-v3 installation contract exposes DWH bindings only
- schema-v3 binding resolution ignores external semantic env vars instead of sourcing runtime semantics from them
- runtime rendering for schema v3 emits backend-owned internal semantic endpoints:
- Qdrant: `http://qdrant:6333`
- Embedding: `http://embedding:11434`
- Model: `qwen3-embedding:0.6b`
- Dimensions: `1024`
- internal semantic URLs are validated to allow only `qdrant` / `embedding` / `localhost` / loopback hosts
- DWH transport/runtime behavior remains unchanged
## Self-review
- Confirmed schema-v3 contracts/docs no longer advertise VECTOR or EMBEDDING installation variables.
- Confirmed schema-v3 runtime output ignores injected external semantic endpoints from env bindings.
- Confirmed semantic endpoints are rendered only in the ephemeral backend-owned harness config path.
- Confirmed type wiring is explicit from `AppConfig` → `ThtRunner` → runtime renderer.
## Concerns
- Host validation currently permits both `http` and `https` on the allowed internal hosts. That keeps the configuration flexible, but if the installation contract intended `http` only, that restriction is not enforced here.
+6
View File
@@ -58,6 +58,12 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
secretRoots: config.workspaceRegistry.secretRoots,
secretsFile: config.secretsFile,
secretFiles: config.secretFiles,
semanticRuntime: {
internalQdrantUrl: config.internalQdrantUrl,
internalEmbeddingUrl: config.internalEmbeddingUrl,
internalEmbeddingModel: config.internalEmbeddingModel,
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
},
});
const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
const hub = deps?.hub ?? new SseHub();
+61
View File
@@ -30,6 +30,10 @@ export interface AppConfig {
legacyWorkspaceMode: boolean;
workspaceDiagnosticTimeoutMs: number;
workspaceRegistry: WorkspaceRegistryConfig;
internalQdrantUrl: string;
internalEmbeddingUrl: string;
internalEmbeddingModel: string;
internalEmbeddingDimensions: number;
}
export const MAX_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS = 10_000;
@@ -101,6 +105,47 @@ function piManagementTimeout(value: string | undefined): number {
return timeout;
}
function loopbackHost(host: string): boolean {
return host === "::1"
|| host === "127.0.0.1"
|| /^127(?:\.\d{1,3}){3}$/.test(host);
}
function internalServiceUrl(
value: string | undefined,
fallback: string,
label: string,
allowedHosts: readonly string[],
): string {
const raw = value ?? fallback;
let parsed: URL;
try {
parsed = new URL(raw);
} catch {
throw new Error(`${label} configuration is invalid`);
}
if (
(parsed.protocol !== "http:" && parsed.protocol !== "https:")
|| parsed.username.length > 0
|| parsed.password.length > 0
|| parsed.pathname !== "/"
|| parsed.search.length > 0
|| parsed.hash.length > 0
|| (!allowedHosts.includes(parsed.hostname) && !loopbackHost(parsed.hostname))
) {
throw new Error(`${label} configuration is invalid`);
}
return parsed.toString().replace(/\/$/, "");
}
function positiveDimension(value: string | undefined, fallback: number): number {
const parsed = Number(value ?? fallback);
if (!Number.isSafeInteger(parsed) || parsed <= 0) {
throw new Error("internal embedding dimensions configuration is invalid");
}
return parsed;
}
export function loadConfig(env: Record<string, string | undefined>): AppConfig {
const authMode = env.AUTH_MODE ?? "none";
if (!(["none", "mock", "upstream"] as const).includes(authMode as AppConfig["authMode"])) {
@@ -208,6 +253,18 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
maxImportEntries: positiveImportLimit(env.THT_WORKSPACE_MAX_IMPORT_ENTRIES, 32),
};
const settingsFile = env.SETTINGS_FILE ?? "data/settings.json";
const internalQdrantUrl = internalServiceUrl(
env.THT_INTERNAL_QDRANT_URL,
"http://qdrant:6333",
"internal Qdrant URL",
["qdrant", "localhost"],
);
const internalEmbeddingUrl = internalServiceUrl(
env.THT_INTERNAL_EMBEDDING_URL,
"http://embedding:11434",
"internal embedding URL",
["embedding", "localhost"],
);
return {
host: env.HOST ?? "127.0.0.1",
port: Number(env.PORT ?? 8787),
@@ -231,5 +288,9 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
legacyWorkspaceMode: legacyWorkspaceMode === "local",
workspaceDiagnosticTimeoutMs: diagnosticTimeout(env.THT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS),
workspaceRegistry,
internalQdrantUrl,
internalEmbeddingUrl,
internalEmbeddingModel: env.THT_INTERNAL_EMBEDDING_MODEL ?? "qwen3-embedding:0.6b",
internalEmbeddingDimensions: positiveDimension(env.THT_INTERNAL_EMBEDDING_DIMENSIONS, 1024),
};
}
+8 -1
View File
@@ -9,7 +9,12 @@ import { parseAllDocuments } from "yaml";
import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js";
import { secretValue, type SecretBundleConfig } from "../config/secret-bundle.js";
import { resolveRuntimeBindings } from "../workspaces/bindings.js";
import { renderRuntimeConfig, type RuntimeInstallationOverlay, type RuntimePaths } from "../workspaces/runtime-renderer.js";
import {
renderRuntimeConfig,
type RuntimeInstallationOverlay,
type RuntimePaths,
type SemanticRuntimeConfig,
} from "../workspaces/runtime-renderer.js";
import { parseWorkspaceYaml } from "../workspaces/schema.js";
export interface ThtConfig extends SecretBundleConfig {
@@ -19,6 +24,7 @@ export interface ThtConfig extends SecretBundleConfig {
dataRoot?: string;
runtimeSnapshotRoot?: string;
secretRoots?: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
}
export interface RuntimeConfigLease {
@@ -190,6 +196,7 @@ export class ThtRunner {
this.runtimePaths(canonical.workspaceId),
canonical,
this.installationOverlay(),
this.cfg.semanticRuntime,
);
const path = this.createRuntimeSnapshot(config);
let released = false;
+1 -4
View File
@@ -97,7 +97,7 @@ export function resolveBinding(
): ResolvedBinding {
const descriptor = validateWorkspaceDescriptor(workspace);
if (descriptor.workspace.schema_version === 3 && role !== "DWH") {
throw new Error("Schema version 3 semantic bindings are not supported by the legacy installation contract");
return { transport: "rest_api", values: {}, missing: [] };
}
const contract = buildInstallationContract(descriptor);
@@ -143,9 +143,6 @@ export function resolveRuntimeBindings(
secretRoots: readonly string[],
): RuntimeBindings {
const descriptor = validateWorkspaceDescriptor(workspace);
if (descriptor.workspace.schema_version === 3) {
throw new Error("Schema version 3 semantic runtime bindings are not supported before the internal Qdrant/Ollama runtime lands");
}
return {
dwh: resolveBinding(descriptor, "DWH", env, secretRoots),
+79 -7
View File
@@ -20,6 +20,20 @@ export interface RuntimeInstallationOverlay {
profile?: unknown;
}
export interface SemanticRuntimeConfig {
internalQdrantUrl: string;
internalEmbeddingUrl: string;
internalEmbeddingModel: string;
internalEmbeddingDimensions: number;
}
const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
internalQdrantUrl: "http://qdrant:6333",
internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingModel: "qwen3-embedding:0.6b",
internalEmbeddingDimensions: 1024,
};
function seconds(timeoutMs: number | undefined): number | undefined {
return timeoutMs === undefined ? undefined : Math.max(1, Math.ceil(timeoutMs / 1_000));
}
@@ -85,13 +99,77 @@ export function renderRuntimeConfig(
paths: RuntimePaths,
identity?: RuntimeIdentity,
installation: RuntimeInstallationOverlay = {},
semanticRuntime: SemanticRuntimeConfig = DEFAULT_SEMANTIC_RUNTIME,
): string {
const descriptor = validateWorkspaceDescriptor(workspace);
const contract = buildInstallationContract(descriptor);
const name = (role: "DWH" | "VECTOR" | "EMBEDDING", suffix: string) => {
const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix);
if (!variable) throw new Error(`workspace contract is missing ${role}_${suffix}`);
return variable.name;
};
if (descriptor.workspace.schema_version !== 2) {
if (descriptor.workspace.schema_version === 1) {
throw new Error("Workspace descriptor requires explicit migration to schema version 2");
}
throw new Error("Schema version 3 runtime rendering is unsupported until the internal semantic runtime is implemented");
if (bindings.dwh.missing.length > 0) {
throw new Error("runtime configuration requires complete bindings");
}
const dwhRest = bindings.dwh.transport === "rest_api";
const dwhIdentity = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
const database = bindings.dwh.transport === "postgres_direct"
? { ...legacyDirectConnection(bindings.dwh, {
host: name("DWH", "HOST"),
port: name("DWH", "PORT"),
user: name("DWH", "USER"),
passwordFile: name("DWH", "PASSWORD_FILE"),
tlsCaFile: name("DWH", "TLS_CA_FILE"),
}, dwhIdentity), transport: "direct" }
: placeholderConnection(dwhIdentity);
const renderedV3: Record<string, unknown> = {
...(identity ? {
runtime_identity: {
workspace_id: identity.workspaceId,
workspace_revision: identity.workspaceRevision,
source_identity: `workspace://${identity.workspaceId}`,
},
} : {}),
...(installation.session_storage === undefined
? {} : { session_storage: installation.session_storage }),
...(installation.profile === undefined ? {} : { profile: installation.profile }),
language: descriptor.workspace.language,
database,
embeddings: {
provider: "ollama_internal",
base_url: semanticRuntime.internalEmbeddingUrl,
model: semanticRuntime.internalEmbeddingModel,
dimensions: semanticRuntime.internalEmbeddingDimensions,
},
resources: {
vector: {
engine: "qdrant",
base_url: semanticRuntime.internalQdrantUrl,
collection: descriptor.semantic_index.vector_store.collection,
},
},
roots: paths,
paths,
};
if (bindings.dwh.transport === "postgres_direct") {
renderedV3.dwh = { type: "postgres_direct", connection: database };
} else if (dwhRest) {
renderedV3.rest = legacyRestEndpoint(bindings.dwh, {
baseUrl: name("DWH", "BASE_URL"),
apiKeyFile: name("DWH", "API_KEY_FILE"),
tlsCaFile: name("DWH", "TLS_CA_FILE"),
}, descriptor.diagnostics?.dwh_rest?.auth !== "none");
renderedV3.database = placeholderConnection(dwhIdentity);
renderedV3.dwh = { type: "thoth_rest", database: dwhIdentity, endpoint: renderedV3.rest };
} else {
throw new Error("ssh_tunnel runtime configuration requires a diagnostic tunnel");
}
return stringify(renderedV3, { lineWidth: 0, sortMapEntries: false });
}
const canonical = descriptor as WorkspaceV2;
@@ -99,12 +177,6 @@ export function renderRuntimeConfig(
throw new Error("runtime configuration requires complete bindings");
}
const contract = buildInstallationContract(canonical);
const name = (role: "DWH" | "VECTOR" | "EMBEDDING", suffix: string) => {
const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix);
if (!variable) throw new Error(`workspace contract is missing ${role}_${suffix}`);
return variable.name;
};
const dwhIdentity = { database: canonical.dwh.database, schema: canonical.dwh.schema };
const vectorIdentity = {
database: canonical.semantic_index.vector_store.database ?? canonical.dwh.database,
+19
View File
@@ -37,10 +37,29 @@ test("loadConfig keeps local development defaults", () => {
maxImportBytes: 10 * 1024 * 1024,
maxImportEntries: 32,
},
internalQdrantUrl: "http://qdrant:6333",
internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingModel: "qwen3-embedding:0.6b",
internalEmbeddingDimensions: 1024,
});
expect(loadConfig({}).dataRoot).toBeUndefined();
});
test("loadConfig accepts only the allowed internal semantic runtime hosts", () => {
expect(loadConfig({
THT_INTERNAL_QDRANT_URL: "http://localhost:6333",
THT_INTERNAL_EMBEDDING_URL: "http://127.0.0.1:11434",
})).toMatchObject({
internalQdrantUrl: "http://localhost:6333",
internalEmbeddingUrl: "http://127.0.0.1:11434",
});
expect(() => loadConfig({ THT_INTERNAL_QDRANT_URL: "http://qdrant.internal:6333" }))
.toThrow(/internal.*qdrant|host validation|invalid/i);
expect(() => loadConfig({ THT_INTERNAL_EMBEDDING_URL: "http://example.com:11434" }))
.toThrow(/internal.*embedding|host validation|invalid/i);
});
test("loadConfig enables the legacy workspace request only through explicit local mode", () => {
expect(loadConfig({ THT_LEGACY_WORKSPACE_MODE: "local" }).legacyWorkspaceMode).toBe(true);
@@ -1,6 +1,11 @@
import { expect, test } from "vitest";
import { parse } from "yaml";
import { renderRuntimeConfig, type RuntimeBindings, type RuntimePaths } from "../src/workspaces/runtime-renderer.js";
import {
renderRuntimeConfig,
type RuntimeBindings,
type RuntimePaths,
type SemanticRuntimeConfig,
} from "../src/workspaces/runtime-renderer.js";
import { supportsSessionRuntime } from "../src/workspaces/bindings.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
@@ -58,6 +63,12 @@ const paths: RuntimePaths = {
artifacts: "/data/workspaces/psd-clinical/artifacts",
indexes: "/data/workspaces/psd-clinical/indexes",
};
const semanticRuntime: SemanticRuntimeConfig = {
internalQdrantUrl: "http://qdrant:6333",
internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingModel: "qwen3-embedding:0.6b",
internalEmbeddingDimensions: 1024,
};
const legacyWorkspace = parseWorkspaceYaml(`workspace:
schema_version: 1
id: psd-clinical
@@ -174,8 +185,56 @@ test("refuses to render a v1 descriptor until an explicit migration creates v2",
test("fails closed for v3 runtime rendering and session support", () => {
expect(supportsSessionRuntime(directBindings)).toBe(true);
expect(() => renderRuntimeConfig(workspaceV3, directBindings, paths))
.toThrow(/unsupported|schema version 3|qdrant|ollama_internal/i);
const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, undefined, {}, semanticRuntime));
expect(rendered.resources).toMatchObject({
vector: {
engine: "qdrant",
base_url: "http://qdrant:6333",
collection: "psd-clinical",
},
});
expect(rendered.embeddings).toMatchObject({
provider: "ollama_internal",
base_url: "http://embedding:11434",
model: "qwen3-embedding:0.6b",
dimensions: 1024,
});
});
test("schema v3 runtime rendering never exposes external semantic endpoints from bindings", () => {
const rendered = parse(renderRuntimeConfig(workspaceV3, {
...directBindings,
vector: {
transport: "rest_api",
missing: [],
values: {
THT_WS_PSD_CLINICAL_VECTOR_BASE_URL: "https://vector.example.test",
THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE: "/run/secrets/vector-api-key",
},
},
embedding: {
transport: "rest_api",
missing: [],
values: {
THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL: "https://embedding.example.test",
},
},
}, paths, undefined, {}, semanticRuntime));
expect(rendered.resources.vector).toMatchObject({
engine: "qdrant",
base_url: "http://qdrant:6333",
collection: "psd-clinical",
});
expect(rendered.embeddings).toMatchObject({
provider: "ollama_internal",
base_url: "http://embedding:11434",
model: "qwen3-embedding:0.6b",
dimensions: 1024,
});
expect(JSON.stringify(rendered)).not.toContain("vector.example.test");
expect(JSON.stringify(rendered)).not.toContain("embedding.example.test");
});
test("omits direct TLS fields when binding validation did not retain a file path", () => {
+30 -4
View File
@@ -222,9 +222,35 @@ test("never treats a vector reader credential as the optional writer binding", (
test("fails closed for v3 external semantic bindings", () => {
expect(() => resolveBinding(workspaceV3, "VECTOR", {}, ["/run/secrets"]))
.toThrow(/unsupported|schema version 3|semantic/i);
.not.toThrow();
expect(() => resolveBinding(workspaceV3, "EMBEDDING", {}, ["/run/secrets"]))
.toThrow(/unsupported|schema version 3|semantic/i);
expect(() => resolveRuntimeBindings(workspaceV3, {}, ["/run/secrets"]))
.toThrow(/unsupported|schema version 3|semantic/i);
.not.toThrow();
expect(() => resolveRuntimeBindings(workspaceV3, {
THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT: "rest_api",
THT_WS_PSD_CLINICAL_VECTOR_BASE_URL: "https://vector.example.test",
THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL: "https://embedding.example.test",
THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE: "/run/secrets/vector-api-key",
}, ["/run/secrets"])).not.toThrow();
});
test("schema v3 ignores external semantic binding variables and reports only DWH requirements", () => {
const password = secretPath("dwh-password");
const bindings = resolveRuntimeBindings(workspaceV3, {
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal",
THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
THT_WS_PSD_CLINICAL_DWH_USER: "reader",
THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: password.path,
THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT: "rest_api",
THT_WS_PSD_CLINICAL_VECTOR_BASE_URL: "https://vector.example.test",
THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE: "/run/secrets/vector-api-key",
THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL: "https://embedding.example.test",
THT_WS_PSD_CLINICAL_EMBEDDING_API_KEY_FILE: "/run/secrets/embedding-api-key",
}, [password.root]);
expect(bindings.dwh.missing).toEqual([]);
expect(bindings.vector.missing).toEqual([]);
expect(bindings.embedding.missing).toEqual([]);
expect(bindings.vector.values).toEqual({});
expect(bindings.embedding.values).toEqual({});
});
+14
View File
@@ -97,6 +97,20 @@ test("renders English UI headings and workspace-language Italian prose", () => {
expect(docs.envExample).toContain("THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT=");
});
test("schema v3 installation contracts expose only DWH bindings and no semantic variables", () => {
const contract = buildInstallationContract(workspaceV3);
const names = contract.variables.map((variable) => variable.name);
const docs = renderWorkspaceDocs(workspaceV3);
expect(contract.variables.every((variable) => variable.role === "DWH")).toBe(true);
expect(names).toContain("THT_WS_PSD_CLINICAL_DWH_TRANSPORT");
expect(names.some((name) => /_VECTOR_|_EMBEDDING_/.test(name))).toBe(false);
expect(docs.envExample).not.toContain("_VECTOR_");
expect(docs.envExample).not.toContain("_EMBEDDING_");
expect(docs.markdown).not.toContain("Vector store");
expect(docs.markdown).not.toContain("Embedding service");
});
test("renders the vector store identity and creates writer credentials only when declared", () => {
const writerWorkspace = parseWorkspaceYaml(`workspace:
schema_version: 2