feat: bind evidence credentials through local files

This commit is contained in:
2026-08-09 19:03:36 +02:00
parent c7a369f436
commit 7126b567b0
11 changed files with 839 additions and 18 deletions
+45 -3
View File
@@ -10,11 +10,17 @@ import {
type WorkspaceDescriptor,
} from "./schema.js";
export interface ResolvedEvidenceBinding {
values: Record<string, string>;
missing: string[];
}
export interface RuntimeBindings {
dwh: ResolvedBinding;
vector: ResolvedBinding;
vectorWriter: ResolvedBinding;
embedding: ResolvedBinding;
evidence: ResolvedEvidenceBinding;
}
export interface ResolvedBinding {
@@ -71,7 +77,7 @@ function isSafeSecretFile(path: string, secretRoots: readonly string[]): boolean
function requiredSuffixes(
workspace: WorkspaceDescriptor,
role: InstallationRole,
role: Exclude<InstallationRole, "EVIDENCE">,
transport: DwhTransport | VectorTransport,
): readonly InstallationSuffix[] {
if (role === "EMBEDDING") return EMBEDDING_REQUIRED_SUFFIXES;
@@ -91,7 +97,7 @@ function requiredSuffixes(
*/
export function resolveBinding(
workspace: WorkspaceDescriptor,
role: InstallationRole,
role: Exclude<InstallationRole, "EVIDENCE">,
env: NodeJS.ProcessEnv,
secretRoots: readonly string[],
): ResolvedBinding {
@@ -136,6 +142,39 @@ export function resolveBinding(
return { transport: selectedTransport, values, missing };
}
/** Resolve descriptor-selected Evidence credentials without reading any secret file contents. */
export function resolveEvidenceBinding(
workspace: WorkspaceDescriptor,
env: NodeJS.ProcessEnv,
secretRoots: readonly string[],
): ResolvedEvidenceBinding {
const descriptor = validateWorkspaceDescriptor(workspace);
const variables = buildInstallationContract(descriptor).variables
.filter((variable) => variable.role === "EVIDENCE");
if (variables.length === 0) return { values: {}, missing: [] };
const source = "evidence" in descriptor ? descriptor.evidence?.source : undefined;
const required = new Set<InstallationSuffix>(
source?.type === "http"
? ["SIGNED_URLS_FILE"]
: source?.type === "s3"
? ["ACCESS_KEY_FILE", "SECRET_KEY_FILE"]
: [],
);
const values: Record<string, string> = {};
const missing: string[] = [];
for (const variable of variables) {
const value = env[variable.name];
const present = value !== undefined && value.trim() !== "";
const safe = present && isSafeSecretFile(value, secretRoots);
if ((required.has(variable.suffix) && !present) || (present && !safe)) {
missing.push(variable.name);
}
if (safe) values[variable.name] = value;
}
return { values, missing };
}
/** Resolve all runtime roles together so optional writer credentials cannot be smuggled into reader bindings. */
export function resolveRuntimeBindings(
workspace: WorkspaceDescriptor,
@@ -149,6 +188,7 @@ export function resolveRuntimeBindings(
vector: resolveBinding(descriptor, "VECTOR", env, secretRoots),
vectorWriter: resolveBinding(descriptor, "VECTOR_WRITER", env, secretRoots),
embedding: resolveBinding(descriptor, "EMBEDDING", env, secretRoots),
evidence: resolveEvidenceBinding(descriptor, env, secretRoots),
};
}
@@ -157,5 +197,7 @@ export function resolveRuntimeBindings(
* session runtime has no tunnel owner. Keep activation fail-closed until that lifecycle exists.
*/
export function supportsSessionRuntime(bindings: RuntimeBindings): boolean {
return bindings.dwh.transport !== "ssh_tunnel" && bindings.vector.transport !== "ssh_tunnel";
return bindings.dwh.transport !== "ssh_tunnel"
&& bindings.vector.transport !== "ssh_tunnel"
&& (bindings.evidence?.missing.length ?? 0) === 0;
}
+28 -4
View File
@@ -1,7 +1,7 @@
import { validateWorkspaceDescriptor } from "./schema.js";
import type { DwhTransport, VectorTransport, WorkspaceDescriptor } from "./schema.js";
export type InstallationRole = "DWH" | "VECTOR" | "VECTOR_WRITER" | "EMBEDDING";
export type InstallationRole = "DWH" | "VECTOR" | "VECTOR_WRITER" | "EMBEDDING" | "EVIDENCE";
export type InstallationSuffix =
| "TRANSPORT"
| "HOST"
@@ -17,7 +17,11 @@ export type InstallationSuffix =
| "SSH_PRIVATE_KEY_FILE"
| "SSH_KNOWN_HOSTS_FILE"
| "SSH_TARGET_HOST"
| "SSH_TARGET_PORT";
| "SSH_TARGET_PORT"
| "SIGNED_URLS_FILE"
| "ACCESS_KEY_FILE"
| "SECRET_KEY_FILE"
| "SESSION_TOKEN_FILE";
type ConnectorTransport = DwhTransport | VectorTransport;
@@ -119,6 +123,25 @@ function connectorVariables(
];
}
function evidenceVariables(
namespace: string,
workspace: WorkspaceDescriptor,
): InstallationVariable[] {
if (!("evidence" in workspace) || workspace.evidence === undefined) return [];
const source = workspace.evidence.source;
if (source.type === "http" && source.authentication === "signed_urls_file") {
return [createVariable(namespace, "EVIDENCE", "SIGNED_URLS_FILE")];
}
if (source.type === "s3" && source.credentials === "static_files") {
return [
createVariable(namespace, "EVIDENCE", "ACCESS_KEY_FILE"),
createVariable(namespace, "EVIDENCE", "SECRET_KEY_FILE"),
createVariable(namespace, "EVIDENCE", "SESSION_TOKEN_FILE"),
];
}
return [];
}
export function buildInstallationContract(workspace: WorkspaceDescriptor): InstallationContract {
const descriptor = validateWorkspaceDescriptor(workspace);
const namespace = namespaceFor(descriptor);
@@ -143,6 +166,7 @@ export function buildInstallationContract(workspace: WorkspaceDescriptor): Insta
...(descriptor.workspace.schema_version === 2
? EMBEDDING_SUFFIXES.map((suffix) => createVariable(namespace, "EMBEDDING", suffix))
: []),
...evidenceVariables(namespace, descriptor),
],
};
}
@@ -179,10 +203,10 @@ export function renderWorkspaceDocs(workspace: WorkspaceDescriptor): { envExampl
"",
"Use the following UI fields as installation bindings. Secret fields always contain file paths, never secret values.",
"",
...(["DWH", "VECTOR", "VECTOR_WRITER", "EMBEDDING"] as const)
...(["DWH", "VECTOR", "VECTOR_WRITER", "EMBEDDING", "EVIDENCE"] as const)
.filter((role) => variablesByRole.has(role))
.flatMap((role) => [
`## ${role === "DWH" ? "Data warehouse" : role === "VECTOR" ? "Vector store" : role === "VECTOR_WRITER" ? "Vector writer" : "Embedding service"}`,
`## ${role === "DWH" ? "Data warehouse" : role === "VECTOR" ? "Vector store" : role === "VECTOR_WRITER" ? "Vector writer" : role === "EMBEDDING" ? "Embedding service" : "Evidence"}`,
"",
...(variablesByRole.get(role) ?? []).map((variable) => (
`- \`${variable.name}\`${variable.transports ? ` (for: ${variable.transports.join(", ")})` : ""}`
+14 -3
View File
@@ -22,6 +22,7 @@ export interface Diagnostic {
level: "error" | "warning" | "info";
code: WorkspaceErrorCode | "binding_ok";
field?: string;
variable?: string;
message: string;
}
@@ -640,9 +641,19 @@ async function diagnoseSchemaV3Workspace(
timeoutMs: number,
semanticRuntime: SemanticRuntimeConfig,
): Promise<WorkspaceDiagnostics> {
const diagnostics = [...bindings.dwh.missing]
.sort()
.map((field) => diagnosticError("binding_missing", field));
const evidenceField = descriptor.evidence?.source.type === "http"
? "evidence.source.authentication"
: descriptor.evidence?.source.type === "s3"
? "evidence.source.credentials"
: undefined;
const evidenceDiagnostics = [...bindings.evidence.missing].sort().map((variable): Diagnostic => ({
...diagnosticError("binding_missing", evidenceField),
variable,
}));
const diagnostics = [
...[...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field)),
...evidenceDiagnostics,
];
if (diagnostics.length > 0) {
return { activatable: false, diagnostics };
}
+96
View File
@@ -2797,3 +2797,99 @@ test("POST /sessions bootstrap failure emits only a fixed recovery message", asy
expect(clientOutput).not.toContain("DO_NOT_LEAK");
expect(clientOutput).not.toContain("/srv/private/model-key");
});
test.each([
{ mode: "missing signed Evidence file", evidence: true, safe: false, expectedStatus: 409, reachesReadiness: false },
{ mode: "safe signed Evidence file", evidence: true, safe: true, expectedStatus: 503, reachesReadiness: true },
{ mode: "no Evidence descriptor", evidence: false, safe: false, expectedStatus: 503, reachesReadiness: true },
])("real buildApp admission handles $mode before Pi spawn", async ({
evidence, safe, expectedStatus, reachesReadiness,
}) => {
const root = mkdtempSync(path.join(tmpdir(), "thoth-evidence-admission-"));
const signedFile = path.join(root, "signed-urls.json");
writeFileSync(signedFile, '["CANARY-SIGNED-QUERY"]');
const variable = "THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE";
const previous = {
transport: process.env.THT_WS_PSD_CLINICAL_DWH_TRANSPORT,
baseUrl: process.env.THT_WS_PSD_CLINICAL_DWH_BASE_URL,
signed: process.env[variable],
};
process.env.THT_WS_PSD_CLINICAL_DWH_TRANSPORT = "rest_api";
process.env.THT_WS_PSD_CLINICAL_DWH_BASE_URL = "https://dwh.example.test";
if (safe) process.env[variable] = signedFile;
else delete process.env[variable];
const descriptor = {
...operationalWorkspace("psd-clinical"),
dwh: {
...operationalWorkspace("psd-clinical").dwh,
supported_transports: ["rest_api"],
},
diagnostics: {
dwh_rest: {
method: "GET", path: "/health", auth: "none",
response: { database: "database", schema: "schema" },
},
},
...(evidence ? {
evidence: {
source: {
type: "http",
uris: ["https://evidence.example.test/guide.md"],
authentication: "signed_urls_file",
connect_timeout_ms: 5_000,
read_timeout_ms: 30_000,
max_bytes: 10 * 1024 * 1024,
max_redirects: 5,
allow_private_hosts: false,
max_cache_bytes: 64 * 1024 * 1024,
},
policy: { max_chunk_chars: 4_000, retain_published_generations: 3 },
},
} : {}),
} as any;
const canonicalBefore = JSON.stringify(descriptor);
const ensure = vi.fn(async () => ({ ok: false, code: "workspace_not_activatable" as const }));
const createFor = vi.fn();
const abort = vi.fn(async () => {});
const revision = {
id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40),
snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/psd-clinical.yaml`,
state: "operational" as const,
};
try {
const app = buildRealApp(loadConfig({
THT_HARNESS_DIR: "../harness",
THT_WORKSPACE_SECRET_ROOTS: root,
}), {
thtRunner: { sessionNew: vi.fn(), searchPack: async () => {} } as any,
readiness: { ensure } as any,
mgr: { get: () => undefined, createFor } as any,
getSettings: () => ({ workspace: "psd-clinical" }) as any,
workspaceRegistry: {
acquireSessionRevision: vi.fn(async () => ({
workspace: descriptor, revision, abort, markPersisted: vi.fn(async () => {}),
})),
} as any,
});
const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
expect(response.statusCode).toBe(expectedStatus);
expect(ensure).toHaveBeenCalledTimes(reachesReadiness ? 1 : 0);
expect(createFor).not.toHaveBeenCalled();
expect(JSON.stringify(descriptor)).toBe(canonicalBefore);
expect(revision.commit).toBe("a".repeat(40));
expect(response.body).not.toContain("CANARY-SIGNED-QUERY");
} finally {
const restore = (name: string, value: string | undefined) => {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
};
restore("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", previous.transport);
restore("THT_WS_PSD_CLINICAL_DWH_BASE_URL", previous.baseUrl);
restore(variable, previous.signed);
rmSync(root, { recursive: true, force: true });
}
});
+117 -2
View File
@@ -1,8 +1,8 @@
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test } from "vitest";
import { resolveBinding, resolveRuntimeBindings } from "../src/workspaces/bindings.js";
import { resolveBinding, resolveEvidenceBinding, resolveRuntimeBindings, supportsSessionRuntime } from "../src/workspaces/bindings.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
const workspace = parseWorkspaceYaml(`workspace:
@@ -254,3 +254,118 @@ test("schema v3 ignores external semantic binding variables and reports only DWH
expect(bindings.vector.values).toEqual({});
expect(bindings.embedding.values).toEqual({});
});
function withEvidence(source: Record<string, unknown>) {
return parseWorkspaceYaml(`workspace:
schema_version: 3
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
semantic_index:
vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine }
embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 }
llm_policy: { allowed: [zai/glm-5.2] }
evidence:
source: ${JSON.stringify(source)}
`);
}
const evidenceVariable = (suffix: string) => `THT_WS_PSD_CLINICAL_EVIDENCE_${suffix}`;
test.each([
{ type: "filesystem", uri: "workspace-content/psd-clinical/evidence" },
{ type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" },
{ type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" },
])("does not resolve Evidence variables for $type modes without file credentials", (source) => {
expect(resolveEvidenceBinding(withEvidence(source), {
[evidenceVariable("SIGNED_URLS_FILE")]: "/CANARY/http",
[evidenceVariable("ACCESS_KEY_FILE")]: "/CANARY/access",
}, ["/run/secrets"])).toEqual({ values: {}, missing: [] });
});
test("requires only a safe HTTP signed-URL file and never reads its contents", () => {
const signed = secretPath("evidence-signed-urls");
writeFileSync(signed.path, "CANARY-SIGNED-URL-CONTENT");
const source = withEvidence({
type: "http",
uris: ["https://evidence.example.test/guide.md"],
authentication: "signed_urls_file",
});
const variable = evidenceVariable("SIGNED_URLS_FILE");
expect(resolveEvidenceBinding(source, {}, [signed.root]).missing).toEqual([variable]);
const resolved = resolveEvidenceBinding(source, {
[variable]: signed.path,
[evidenceVariable("ACCESS_KEY_FILE")]: signed.path,
}, [signed.root]);
expect(resolved).toEqual({ values: { [variable]: signed.path }, missing: [] });
expect(JSON.stringify(resolved)).not.toContain("CANARY-SIGNED-URL-CONTENT");
});
test("requires S3 access and secret files together while accepting an optional safe session token", () => {
const access = secretPath("evidence-access");
const secret = secretPath("evidence-secret");
const token = secretPath("evidence-token");
const source = withEvidence({
type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files",
});
const env = {
[evidenceVariable("ACCESS_KEY_FILE")]: access.path,
[evidenceVariable("SECRET_KEY_FILE")]: secret.path,
[evidenceVariable("SESSION_TOKEN_FILE")]: token.path,
[evidenceVariable("SIGNED_URLS_FILE")]: access.path,
};
expect(resolveEvidenceBinding(source, {
[evidenceVariable("ACCESS_KEY_FILE")]: access.path,
}, [access.root]).missing).toEqual([evidenceVariable("SECRET_KEY_FILE")]);
expect(resolveEvidenceBinding(source, env, [access.root, secret.root, token.root])).toEqual({
values: {
[evidenceVariable("ACCESS_KEY_FILE")]: access.path,
[evidenceVariable("SECRET_KEY_FILE")]: secret.path,
[evidenceVariable("SESSION_TOKEN_FILE")]: token.path,
},
missing: [],
});
});
test("rejects relative, missing, directory, unreadable, and escaping symlink Evidence paths", () => {
const allowed = secretPath("valid");
const outside = secretPath("outside");
const directory = join(allowed.root, "directory");
mkdirSync(directory);
const link = join(allowed.root, "escape");
symlinkSync(outside.path, link);
const unreadable = join(allowed.root, "unreadable");
writeFileSync(unreadable, "secret");
chmodSync(unreadable, 0o000);
const source = withEvidence({
type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file",
});
const variable = evidenceVariable("SIGNED_URLS_FILE");
for (const path of ["relative", join(allowed.root, "missing"), directory, unreadable, link]) {
expect(resolveEvidenceBinding(source, { [variable]: path }, [allowed.root])).toEqual({
values: {}, missing: [variable],
});
}
chmodSync(unreadable, 0o600);
});
test("includes Evidence binding completeness in session runtime support without changing v3 compatibility", () => {
const unsigned = resolveRuntimeBindings(workspaceV3, {}, ["/run/secrets"]);
expect(unsigned.evidence).toEqual({ values: {}, missing: [] });
expect(supportsSessionRuntime(unsigned)).toBe(true);
const signed = resolveRuntimeBindings(withEvidence({
type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file",
}), {}, ["/run/secrets"]);
expect(signed.evidence.missing).toEqual([evidenceVariable("SIGNED_URLS_FILE")]);
expect(supportsSessionRuntime(signed)).toBe(false);
});
+59
View File
@@ -190,6 +190,7 @@ llm_policy:
missing: [],
values: { THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL: "https://embedding.internal" },
},
evidence: { missing: [], values: {} },
};
const writerVariables = buildInstallationContract(writerWorkspace).variables
@@ -326,3 +327,61 @@ test("v3 installation contract omits external vector and embedding bindings", ()
expect(names.some((name) => name.includes("_EMBEDDING_"))).toBe(false);
expect(renderWorkspaceDocs(workspaceV3).markdown).not.toContain("Embedding service");
});
test.each([
{
mode: "signed HTTP",
source: {
type: "http", uris: ["https://evidence.example.test/guide.md"],
authentication: "signed_urls_file",
},
expected: ["THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE"],
},
{
mode: "static S3",
source: { type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files" },
expected: [
"THT_WS_PSD_CLINICAL_EVIDENCE_ACCESS_KEY_FILE",
"THT_WS_PSD_CLINICAL_EVIDENCE_SECRET_KEY_FILE",
"THT_WS_PSD_CLINICAL_EVIDENCE_SESSION_TOKEN_FILE",
],
},
])("generates source-specific $mode Evidence file bindings", ({ source, expected }) => {
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
const contract = buildInstallationContract(descriptor);
const evidence = contract.variables.filter((variable) => variable.role === "EVIDENCE");
expect(contract.namespace).toBe("PSD_CLINICAL");
expect(evidence.map((variable) => variable.name)).toEqual(expected);
expect(evidence.every((variable) => variable.secret)).toBe(true);
expect(renderWorkspaceDocs(descriptor).envExample).not.toContain("CANARY-SECRET");
});
test.each([
{ type: "filesystem", uri: "workspace-content/psd-clinical/evidence" },
{ type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" },
{ type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" },
])("omits Evidence installation variables for $type modes without file credentials", (source) => {
const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`);
expect(buildInstallationContract(descriptor).variables.some((variable) => variable.role === "EVIDENCE"))
.toBe(false);
});
function renderWorkspaceWithoutEvidence(): string {
return `workspace:
schema_version: 3
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
semantic_index:
vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine }
embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 }
llm_policy: { allowed: [zai/glm-5.2] }
`;
}
@@ -141,6 +141,7 @@ const bindings: RuntimeBindings = {
THT_WS_PSD_CLINICAL_EMBEDDING_TLS_CA_FILE: "/run/secrets/embedding-ca",
},
},
evidence: { missing: [], values: {} },
};
const writerBindings: RuntimeBindings = {
@@ -190,6 +191,7 @@ const bindingsV3: RuntimeBindings = {
vector: { transport: "rest_api", missing: [], values: {} },
vectorWriter: { transport: "rest_api", missing: [], values: {} },
embedding: { transport: "rest_api", missing: [], values: {} },
evidence: { missing: [], values: {} },
};
function successfulAdapters(overrides: Partial<DiagnosticAdapters> = {}): DiagnosticAdapters {
@@ -960,3 +962,55 @@ test("attempts bounded cleanup when a timed-out write may already have created t
expect(adapters.removeDiagnosticRecord).toHaveBeenCalledOnce();
expect(result.activatable).toBe(false);
});
test.each([
{
source: {
type: "http", uris: ["https://evidence.example.test/guide.md"],
authentication: "signed_urls_file",
},
field: "evidence.source.authentication",
variable: "THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE",
},
{
source: { type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files" },
field: "evidence.source.credentials",
variable: "THT_WS_PSD_CLINICAL_EVIDENCE_ACCESS_KEY_FILE",
},
])("reports sanitized v3 Evidence binding diagnostics for $field", async ({ source, field, variable }) => {
const descriptor = parseWorkspaceYaml(`${renderEvidenceWorkspace()}evidence:\n source: ${JSON.stringify(source)}\n`);
const resolved: RuntimeBindings = {
...resolveRuntimeBindings(descriptor, {
[variable]: "CANARY-UNSAFE-RELATIVE-PATH",
}, ["/run/secrets"]),
dwh: bindings.dwh,
};
const result = await createProductionWorkspaceDiagnoser(5_000)(descriptor, resolved, { writeProbe: false });
expect(result).toEqual({
activatable: false,
diagnostics: expect.arrayContaining([expect.objectContaining({
code: "binding_missing", field, variable,
})]),
});
expect(JSON.stringify(result)).not.toContain("CANARY-UNSAFE-RELATIVE-PATH");
});
function renderEvidenceWorkspace(): string {
return `workspace:
schema_version: 3
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: warehouse
schema: datawarehouse
supported_transports: [postgres_direct]
semantic_index:
vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine }
embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 }
llm_policy: { allowed: [zai/glm-5.2] }
`;
}
+17
View File
@@ -384,3 +384,20 @@ evidence:
legacy_source = build_evidence_sources(load_config(legacy))[0]
assert isinstance(legacy_source, FilesystemEvidenceSource)
assert legacy_source.root == (tmp_path / "curated").resolve()
def test_safe_validation_formatter_keeps_location_and_type_without_rejected_input(tmp_path):
workspace = tmp_path / "workspace.yaml"
workspace.write_text("""
dwh:
type: postgres_direct
connection: {database: analytics, schema: public, user: reader}
""")
with pytest.raises(ConfigError) as caught:
load_config(workspace)
message = str(caught.value)
assert "dwh.postgres_direct.connection.password" in message
assert "missing" in message
@@ -0,0 +1,280 @@
import json
import pytest
import yaml
from pydantic import SecretStr
from typer.testing import CliRunner
from tht.adapters.evidence import HttpManifestEvidenceSource
from tht.adapters.factory import build_evidence_sources
from tht.cli import app
from tht.config import ConfigError, load_config
SIGNED_CANARY = "SIGNED-CANARY-QUERY"
ACCESS_CANARY = "ACCESS-CANARY"
SECRET_CANARY = "SECRET-CANARY"
TOKEN_CANARY = "TOKEN-CANARY"
ALL_CANARIES = (SIGNED_CANARY, ACCESS_CANARY, SECRET_CANARY, TOKEN_CANARY)
def raw_runtime(source, *, vector=None):
value = {
"dwh": {
"type": "postgres_direct",
"connection": {
"database": "analytics", "schema": "public", "user": "reader",
"password": "not-a-canary",
},
},
"evidence": {"sources": [source]},
}
if vector is not None:
value["vector"] = vector
return value
def write_config(tmp_path, source, *, vector=None):
path = tmp_path / "runtime.yaml"
path.write_text(yaml.safe_dump(raw_runtime(source, vector=vector)))
return path
def assert_no_canaries(value):
text = str(value)
for canary in ALL_CANARIES:
assert canary not in text
def test_filesystem_config_does_not_touch_a_declared_source_root(tmp_path):
missing = tmp_path / "deliberately-missing"
cfg = load_config(write_config(tmp_path, {"type": "filesystem", "root": str(missing)}))
assert cfg.evidence.sources[0].root == missing
assert cfg.evidence.sources[0].patterns == ["**/*.md"]
assert cfg.evidence.sources[0].max_bytes == 10 * 1024 * 1024
assert not missing.exists()
def test_public_http_urls_are_secret_typed_without_adapter_construction(tmp_path):
cfg = load_config(write_config(tmp_path, {
"type": "http", "urls": ["https://evidence.example.test/guide.md"],
}))
source = cfg.evidence.sources[0]
assert isinstance(source.urls[0], SecretStr)
assert source.transport_urls() == ["https://evidence.example.test/guide.md"]
assert source.connect_timeout == 5
assert source.read_timeout == 30
assert source.max_bytes == 10 * 1024 * 1024
assert source.max_redirects == 5
assert source.allow_private_hosts is False
assert source.max_cache_bytes == 64 * 1024 * 1024
def test_signed_http_file_resolves_in_memory_and_preserves_provenance_order(tmp_path):
signed = [
f"https://evidence.example.test/guide.md?token={SIGNED_CANARY}",
"https://evidence.example.test/runbook.md?signature=second",
]
secret_file = tmp_path / "signed-urls.json"
secret_file.write_text(json.dumps(signed))
cfg = load_config(write_config(tmp_path, {
"type": "http",
"provenance_urls": [
"https://evidence.example.test/guide.md",
"https://evidence.example.test/runbook.md",
],
"signed_urls_file": str(secret_file),
"connect_timeout": 7,
"read_timeout": 41,
"max_bytes": 1234,
"max_redirects": 2,
"allow_private_hosts": True,
"max_cache_bytes": 5678,
}))
source = cfg.evidence.sources[0]
assert all(isinstance(url, SecretStr) for url in source.urls)
assert source.transport_urls() == signed
assert source.provenance_urls == [
"https://evidence.example.test/guide.md",
"https://evidence.example.test/runbook.md",
]
assert (source.connect_timeout, source.read_timeout) == (7, 41)
assert (source.max_bytes, source.max_redirects, source.max_cache_bytes) == (1234, 2, 5678)
assert source.allow_private_hosts is True
assert "signed_urls_file" not in repr(source)
assert_no_canaries(repr(cfg))
assert_no_canaries(cfg.model_dump_json())
adapter = build_evidence_sources(cfg)[0]
assert isinstance(adapter, HttpManifestEvidenceSource)
assert_no_canaries(repr(adapter))
@pytest.mark.parametrize("contents", [
"{malformed", json.dumps({"url": "https://evidence.example.test/guide.md"}),
json.dumps([]), json.dumps(["https://evidence.example.test/guide.md", 3]),
])
def test_signed_http_rejects_malformed_non_list_empty_or_non_string_files(tmp_path, contents):
secret_file = tmp_path / "signed-urls.json"
secret_file.write_text(contents)
path = write_config(tmp_path, {
"type": "http",
"provenance_urls": ["https://evidence.example.test/guide.md"],
"signed_urls_file": str(secret_file),
})
with pytest.raises(ConfigError) as caught:
load_config(path)
assert "signed URL file" in str(caught.value)
assert_no_canaries(caught.value)
def test_signed_http_rejects_missing_and_oversized_files_without_disclosure(tmp_path):
missing = tmp_path / "missing.json"
path = write_config(tmp_path, {
"type": "http",
"provenance_urls": ["https://evidence.example.test/guide.md"],
"signed_urls_file": str(missing),
})
with pytest.raises(ConfigError, match="signed URL file"):
load_config(path)
oversized = tmp_path / "oversized.json"
oversized.write_bytes(b"x" * (1024 * 1024 + 1))
path = write_config(tmp_path, {
"type": "http",
"provenance_urls": ["https://evidence.example.test/guide.md"],
"signed_urls_file": str(oversized),
})
with pytest.raises(ConfigError, match="signed URL file") as caught:
load_config(path)
assert_no_canaries(caught.value)
@pytest.mark.parametrize("provenance,signed", [
(
["https://evidence.example.test/a.md", "https://evidence.example.test/b.md"],
["https://evidence.example.test/b.md?sig=1", "https://evidence.example.test/a.md?sig=2"],
),
(["https://evidence.example.test/a.md"], [
"https://evidence.example.test/a.md?sig=1", "https://evidence.example.test/b.md?sig=2",
]),
(["https://evidence.example.test/a.md"], ["https://evidence.example.test/b.md"]),
(["https://evidence.example.test/a.md"], [f"https://user:{SIGNED_CANARY}@evidence.example.test/a.md"]),
(
["https://evidence.example.test/a.md", "https://evidence.example.test/a.md"],
["https://evidence.example.test/a.md?sig=1", "https://evidence.example.test/a.md?sig=2"],
),
])
def test_signed_http_rejects_reordered_extra_mismatch_userinfo_and_duplicate_provenance(
tmp_path, provenance, signed,
):
secret_file = tmp_path / "signed-urls.json"
secret_file.write_text(json.dumps(signed))
path = write_config(tmp_path, {
"type": "http", "provenance_urls": provenance, "signed_urls_file": str(secret_file),
})
with pytest.raises(ConfigError) as caught:
load_config(path)
assert "evidence.sources.0" in str(caught.value)
assert_no_canaries(caught.value)
def test_s3_ambient_and_static_file_credentials_are_secret_typed(tmp_path):
ambient = load_config(write_config(tmp_path, {
"type": "s3", "bucket": "clinical-evidence", "prefix": "published/",
})).evidence.sources[0]
assert ambient.access_key is None
assert ambient.secret_key is None
assert ambient.session_token is None
assert ambient.max_bytes == 10 * 1024 * 1024
assert ambient.max_objects == 10_000
assert ambient.max_pages == 100
assert ambient.page_size == 1000
files = {}
for name, canary in [
("access_key", ACCESS_CANARY), ("secret_key", SECRET_CANARY),
("session_token", TOKEN_CANARY),
]:
path = tmp_path / name
path.write_text(canary)
files[f"{name}_file"] = str(path)
cfg = load_config(write_config(tmp_path, {
"type": "s3", "bucket": "clinical-evidence", "prefix": "published/",
**files,
"endpoint_url": "https://s3.example.test",
"region": "eu-west-1",
"trusted_endpoint": True,
"allow_private_endpoint": True,
"allow_insecure_endpoint": False,
"max_bytes": 222,
"max_objects": 33,
"max_pages": 4,
"page_size": 5,
}))
source = cfg.evidence.sources[0]
assert all(isinstance(value, SecretStr) for value in (
source.access_key, source.secret_key, source.session_token,
))
assert (source.max_bytes, source.max_objects, source.max_pages, source.page_size) == (222, 33, 4, 5)
assert_no_canaries(repr(cfg))
assert_no_canaries(cfg.model_dump_json())
def test_evidence_policy_defaults_non_defaults_and_unknown_keys(tmp_path):
default = load_config(write_config(tmp_path, {
"type": "filesystem", "root": str(tmp_path / "missing"),
}))
assert default.vector.max_chunk_chars == 4000
assert default.vector.retain_published_generations == 3
explicit = load_config(write_config(tmp_path, {
"type": "filesystem", "root": str(tmp_path / "missing"),
"patterns": ["docs/*.md"], "max_bytes": 99,
}, vector={"max_chunk_chars": 123, "retain_published_generations": 7}))
assert explicit.evidence.sources[0].patterns == ["docs/*.md"]
assert explicit.vector.max_chunk_chars == 123
assert explicit.vector.retain_published_generations == 7
for mutation in [
{"type": "filesystem", "root": str(tmp_path), "unknown": SIGNED_CANARY},
{"type": "http", "urls": ["https://evidence.example.test/a"], "unknown": SIGNED_CANARY},
{"type": "s3", "bucket": "bucket-name", "unknown": SIGNED_CANARY},
]:
with pytest.raises(ConfigError) as caught:
load_config(write_config(tmp_path, mutation))
assert "extra_forbidden" in str(caught.value)
assert_no_canaries(caught.value)
def test_validation_repr_cli_and_exception_output_never_disclose_transport_secrets(tmp_path):
secret_file = tmp_path / "signed-urls.json"
secret_file.write_text(json.dumps([
f"https://evidence.example.test/other.md?token={SIGNED_CANARY}",
]))
path = write_config(tmp_path, {
"type": "http",
"provenance_urls": ["https://evidence.example.test/guide.md"],
"signed_urls_file": str(secret_file),
})
with pytest.raises(ConfigError) as caught:
load_config(path)
assert_no_canaries(caught.value)
valid_file = tmp_path / "valid-signed-urls.json"
valid_file.write_text(json.dumps([
f"https://evidence.example.test/guide.md?token={SIGNED_CANARY}",
]))
valid = write_config(tmp_path, {
"type": "http",
"provenance_urls": ["https://evidence.example.test/guide.md"],
"signed_urls_file": str(valid_file),
})
result = CliRunner().invoke(app, ["config", "check", "--config", str(valid)])
assert result.exit_code == 0
assert_no_canaries(result.stdout)
assert_no_canaries(result.stderr)
+1 -1
View File
@@ -64,7 +64,7 @@ def build_evidence_sources(cfg: Config):
case "http":
sources.append(
HttpManifestEvidenceSource(
[url.get_secret_value() for url in resource.urls],
resource.transport_urls(),
connect_timeout=resource.connect_timeout,
read_timeout=resource.read_timeout,
max_bytes=resource.max_bytes,
+128 -5
View File
@@ -1,5 +1,7 @@
import json
import os
import re
import stat
import warnings
from ipaddress import ip_address
from pathlib import Path
@@ -10,6 +12,7 @@ import yaml
from pydantic import BaseModel, Field, PrivateAttr, SecretStr, ValidationError, model_validator
from tht.config_compat import translate_legacy_config
from tht.ports.evidence import canonical_provenance_uri
_ENV_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
@@ -43,6 +46,51 @@ def _expand_env(value: Any) -> Any:
return value
_MAX_SIGNED_URL_FILE_BYTES = 1024 * 1024
def _resolve_http_signed_url_files(value: Any) -> Any:
"""Resolve only signed HTTP URL arrays, keeping their values out of public errors."""
if isinstance(value, dict):
resolved = {
key: _resolve_http_signed_url_files(item)
for key, item in value.items()
}
if resolved.get("type") != "http" or "signed_urls_file" not in resolved:
return resolved
if "urls" in resolved:
raise ConfigError("HTTP signed URL file cannot be combined with urls")
path_value = resolved.pop("signed_urls_file")
if not isinstance(path_value, str):
raise ConfigError("Invalid signed URL file reference")
path = Path(path_value)
try:
entry = path.lstat()
target = path.stat()
if stat.S_ISLNK(entry.st_mode) or not stat.S_ISREG(target.st_mode):
raise OSError
if target.st_size > _MAX_SIGNED_URL_FILE_BYTES:
raise OSError
with path.open("rb") as stream:
payload = stream.read(_MAX_SIGNED_URL_FILE_BYTES + 1)
if len(payload) > _MAX_SIGNED_URL_FILE_BYTES:
raise OSError
parsed = json.loads(payload.decode("utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
raise ConfigError("Cannot read signed URL file") from exc
if (
not isinstance(parsed, list)
or not parsed
or any(not isinstance(item, str) or not item for item in parsed)
):
raise ConfigError("Invalid signed URL file")
resolved["urls"] = parsed
return resolved
if isinstance(value, list):
return [_resolve_http_signed_url_files(item) for item in value]
return value
def _resolve_secret_files(value: Any) -> Any:
if isinstance(value, dict):
resolved = {key: _resolve_secret_files(item) for key, item in value.items()}
@@ -228,12 +276,15 @@ class FilesystemEvidenceSourceConfig(BaseModel):
patterns: list[str] = ["**/*.md"]
max_bytes: int = Field(default=10 * 1024 * 1024, gt=0)
model_config = {"extra": "forbid"}
class HttpEvidenceSourceConfig(BaseModel):
type: Literal["http"]
# Manifest URLs may contain signed query parameters. Treat the complete transport URL as
# secret-bearing configuration; adapters derive a query-free provenance URI from it.
# Transport URLs are secret-bearing. Signed-file configurations retain only public,
# query-free provenance identities alongside the masked transport values.
urls: list[SecretStr] = Field(min_length=1)
provenance_urls: list[str] | None = Field(default=None, min_length=1)
connect_timeout: float = Field(default=5, gt=0)
read_timeout: float = Field(default=30, gt=0)
max_bytes: int = Field(default=10 * 1024 * 1024, gt=0)
@@ -241,6 +292,38 @@ class HttpEvidenceSourceConfig(BaseModel):
allow_private_hosts: bool = False
max_cache_bytes: int = Field(default=64 * 1024 * 1024, gt=0)
model_config = {"extra": "forbid"}
@model_validator(mode="after")
def validate_provenance_mapping(self):
transport_urls = [url.get_secret_value() for url in self.urls]
try:
canonical = [canonical_provenance_uri(url) for url in transport_urls]
except ValueError as exc:
raise ValueError("HTTP transport URL is invalid") from exc
if any(
urlparse(url).scheme not in ("http", "https") or not urlparse(url).hostname
for url in transport_urls
):
raise ValueError("HTTP transport URL must use http or https")
if self.provenance_urls is None:
return self
try:
provenance = [canonical_provenance_uri(url) for url in self.provenance_urls]
except ValueError as exc:
raise ValueError("HTTP provenance URL is invalid") from exc
if provenance != self.provenance_urls:
raise ValueError("HTTP provenance URLs must be canonical query-free identities")
if len(set(provenance)) != len(provenance):
raise ValueError("HTTP provenance URLs must not repeat")
if len(canonical) != len(provenance) or canonical != provenance:
raise ValueError("Signed HTTP URLs must map one-to-one to provenance URLs in order")
return self
def transport_urls(self) -> list[str]:
"""Expose secret transport values only at the adapter-construction boundary."""
return [url.get_secret_value() for url in self.urls]
class S3EvidenceSourceConfig(BaseModel):
type: Literal["s3"]
@@ -259,6 +342,16 @@ class S3EvidenceSourceConfig(BaseModel):
max_pages: int = Field(default=100, gt=0)
page_size: int = Field(default=1000, gt=0, le=1000)
model_config = {"extra": "forbid"}
@model_validator(mode="after")
def validate_static_credentials(self):
if (self.access_key is None) != (self.secret_key is None):
raise ValueError("S3 access_key and secret_key must be configured together")
if self.session_token is not None and self.access_key is None:
raise ValueError("S3 session_token requires static credentials")
return self
EvidenceSourceConfig = Annotated[
FilesystemEvidenceSourceConfig | HttpEvidenceSourceConfig | S3EvidenceSourceConfig,
@@ -282,6 +375,8 @@ class EvidenceSourcesConfig(BaseModel):
raise ValueError("evidence requires source_root or sources")
return self
model_config = {"extra": "forbid"}
class EmbeddingsConfig(BaseModel):
provider: str = "ollama_internal"
@@ -298,10 +393,12 @@ class EmbeddingsConfig(BaseModel):
class VectorConfig(BaseModel):
max_chunk_chars: int = 4000
max_chunk_chars: int = Field(default=4000, gt=0)
# ACTIVE plus the two most recent rollback generations by default.
retain_published_generations: int = Field(default=3, ge=1)
model_config = {"extra": "forbid"}
class SearchConfig(BaseModel):
rrf_k: int = 60
@@ -392,6 +489,31 @@ def workspace_id_for_config(config: Config, path: Path) -> str:
return workspace_id_from_path(path)
def _format_validation_error(error: ValidationError) -> str:
messages = {
"missing": "required field",
"extra_forbidden": "unknown field",
"greater_than": "value must be greater than the configured bound",
"greater_than_equal": "value must meet the configured lower bound",
"less_than_equal": "value exceeds the configured upper bound",
"literal_error": "unsupported literal value",
"union_tag_invalid": "unsupported discriminator",
"union_tag_not_found": "missing discriminator",
"string_too_short": "string is too short",
"too_short": "collection is too short",
"value_error": "configuration value is invalid",
}
lines = []
for issue in error.errors(include_input=False, include_url=False):
location = ".".join(str(part) for part in issue.get("loc", ())) or "configuration"
error_type = str(issue.get("type", "validation_error"))
message = messages.get(error_type, "invalid configuration value")
if location == "runtime_identity" and error_type == "value_error":
message = "source_identity does not match workspace identity"
lines.append(f"{location} [{error_type}]: {message}")
return "\n".join(lines)
def load_config(path: Path) -> Config:
if not path.exists():
raise ConfigError(f"File di configurazione non trovato: {path}")
@@ -401,7 +523,7 @@ def load_config(path: Path) -> Config:
raise ConfigError(f"Configurazione YAML non valida: {path}") from exc
if not isinstance(raw, dict):
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
expanded = _resolve_secret_files(_expand_env(raw))
expanded = _resolve_secret_files(_resolve_http_signed_url_files(_expand_env(raw)))
_validate_internal_embedding_contract(expanded, path)
_validate_internal_vector_contract(expanded, path)
translated, used_legacy = translate_legacy_config(expanded)
@@ -409,7 +531,8 @@ def load_config(path: Path) -> Config:
try:
cfg = Config.model_validate(translated)
except ValidationError as e:
raise ConfigError(f"Configurazione non valida in {path}:\n{e}") from e
details = _format_validation_error(e)
raise ConfigError(f"Configurazione non valida in {path}:\n{details}") from e
env_profile = os.environ.get("THT_PROFILE")
if env_profile is not None:
if env_profile not in ("server", "workstation"):