From 0724a73300b15430a68db68cf677ace3035d63ea Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 11 Aug 2026 07:13:21 +0200 Subject: [PATCH] fix: serialize registry qdrant lifecycle --- backend/src/workspaces/runtime-renderer.ts | 6 + .../test/workspace-runtime-renderer.test.ts | 62 ++++++++++ .../tests/test_registry_evidence_config.py | 107 ++++++++++++++++-- 3 files changed, 168 insertions(+), 7 deletions(-) diff --git a/backend/src/workspaces/runtime-renderer.ts b/backend/src/workspaces/runtime-renderer.ts index f6499f89..c120464f 100644 --- a/backend/src/workspaces/runtime-renderer.ts +++ b/backend/src/workspaces/runtime-renderer.ts @@ -255,6 +255,12 @@ export function renderRuntimeConfig( ...(installation.profile === undefined ? {} : { profile: installation.profile }), language: descriptor.workspace.language, database, + vectors: { + type: "qdrant", + base_url: semanticRuntime.internalQdrantUrl, + collection: descriptor.semantic_index.vector_store.collection, + collection_lifecycle: identity ? "require_existing" : "create_if_missing", + }, resources: { vector: { engine: "qdrant", diff --git a/backend/test/workspace-runtime-renderer.test.ts b/backend/test/workspace-runtime-renderer.test.ts index bb1fce1b..06608e45 100644 --- a/backend/test/workspace-runtime-renderer.test.ts +++ b/backend/test/workspace-runtime-renderer.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -80,6 +81,10 @@ test("renders only the schema-v3 internal Qdrant and Ollama runtime shape", () = ssl_ca_file: "/run/secrets/dwh-ca.pem", transport: "direct", }, dwh: { type: "postgres_direct" }, + vectors: { + type: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical", + collection_lifecycle: "require_existing", + }, resources: { vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" }, embeddings: { @@ -94,6 +99,63 @@ test("renders only the schema-v3 internal Qdrant and Ollama runtime shape", () = expect(rendered).not.toHaveProperty("vector_rest"); }); +test("keeps create_if_missing for non-registry runtime renders", () => { + const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, undefined, {}, semanticRuntime)); + + expect(rendered.vectors).toMatchObject({ + type: "qdrant", + collection_lifecycle: "create_if_missing", + }); +}); + +test.each(["session", "maintenance"])( + "renders require_existing vectors for %s acquisition", + (mode) => { + const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, { + workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40), + }, {}, semanticRuntime)); + + expect(rendered.vectors).toEqual({ + type: "qdrant", + base_url: "http://qdrant:6333", + collection: "psd-clinical", + collection_lifecycle: "require_existing", + }); + }, +); + +test("session and maintenance renders are byte-identical and bind identically in harness", () => { + const context = { workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40) }; + const passwordFile = evidenceSecretFile("dwh-password", "not-a-canary"); + const bindings = { + ...directBindings, + dwh: { ...directBindings.dwh, values: { + ...directBindings.dwh.values, + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: passwordFile, + } }, + }; + const session = renderRuntimeConfig(workspaceV3, bindings, paths, context, {}, semanticRuntime); + const maintenance = renderRuntimeConfig(workspaceV3, bindings, paths, context, {}, semanticRuntime); + + expect(maintenance).toBe(session); + + const script = ` +import json, sys, tempfile +from pathlib import Path +from tht.config import load_config +from tht.jobs.dwh_pipeline import config_dwh_binding +with tempfile.TemporaryDirectory() as root: + path = Path(root) / "runtime.yaml" + path.write_text(sys.stdin.read()) + print(json.dumps(config_dwh_binding(load_config(path)), sort_keys=True)) +`; + const python = join(process.cwd(), "../harness/.venv/bin/python"); + const run = (yaml: string) => execFileSync(python, ["-c", script], { + cwd: join(process.cwd(), "../harness"), input: yaml, encoding: "utf8", + }).trim(); + expect(run(session)).toBe(run(maintenance)); +}); + test("renders schema-v3 DWH REST without exposing secret contents", () => { const rendered = parse(renderRuntimeConfig(workspaceV3, { dwh: { diff --git a/harness/tests/test_registry_evidence_config.py b/harness/tests/test_registry_evidence_config.py index 718650ef..e70ac8fa 100644 --- a/harness/tests/test_registry_evidence_config.py +++ b/harness/tests/test_registry_evidence_config.py @@ -1,5 +1,7 @@ import json +import subprocess import traceback +from pathlib import Path import pytest import yaml @@ -7,10 +9,11 @@ from pydantic import SecretStr from typer.testing import CliRunner from tht.adapters.evidence import HttpManifestEvidenceSource -from tht.adapters.factory import build_evidence_sources +from tht.adapters.factory import build_evidence_sources, build_vector_store from tht.cli import app from tht.config import ConfigError, load_config from tht.jobs.dwh_pipeline import config_dwh_binding +from tht.ports.vector import VectorStoreError SIGNED_CANARY = "SIGNED-CANARY-QUERY" ACCESS_CANARY = "ACCESS-CANARY" @@ -353,6 +356,71 @@ def test_validation_repr_cli_and_exception_output_never_disclose_transport_secre assert_no_canaries(result.stderr) +def _render_registry_runtime_config(tmp_path) -> str: + """Render the production schema-v3 runtime rather than duplicating its YAML.""" + tmp_path.mkdir(parents=True, exist_ok=True) + backend = Path(__file__).resolve().parents[2] / "backend" + password_file = tmp_path / "dwh-password" + password_file.write_text("not-a-canary") + script = r""" +import { parseWorkspaceYaml } from "__SCHEMA__"; +import { renderRuntimeConfig } from "__RENDERER__"; +const workspace = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Runtime test + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: workspace-semantic + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`); +const bindings = { + dwh: { transport: "postgres_direct", missing: [], values: { + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: process.argv[2], + } }, + evidence: { missing: [], values: {} }, +}; +process.stdout.write(renderRuntimeConfig(workspace, bindings, { + sessions: "/tmp/sessions", artifacts: "/tmp/artifacts", indexes: "/tmp/indexes", +}, { workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40) })); +""" + script = script.replace( + "__SCHEMA__", str(backend / "src/workspaces/schema.ts"), + ).replace("__RENDERER__", str(backend / "src/workspaces/runtime-renderer.ts")) + script_path = tmp_path / "render-runtime.mts" + script_path.write_text(script) + result = subprocess.run( + [str(backend / "node_modules/.bin/tsx"), str(script_path), str(password_file)], + cwd=backend, check=True, capture_output=True, text=True, + ) + return result.stdout + + +def _render_registry_runtime_configs(tmp_path) -> tuple[Path, Path]: + session = tmp_path / "session.yaml" + maintenance = tmp_path / "maintenance.yaml" + session.write_text(_render_registry_runtime_config(tmp_path)) + maintenance.write_text(_render_registry_runtime_config(tmp_path)) + return session, maintenance + + def _qdrant_config_yaml(tmp_path, *, registry: bool) -> dict: value = { "dwh": { @@ -378,14 +446,39 @@ def _qdrant_config_yaml(tmp_path, *, registry: bool) -> dict: return value -@pytest.mark.parametrize("mode", ["session", "maintenance"]) -def test_registry_runtime_configs_require_existing_qdrant_collection(tmp_path, mode): - path = tmp_path / f"{mode}.yaml" - path.write_text(yaml.safe_dump(_qdrant_config_yaml(tmp_path, registry=True))) +@pytest.mark.parametrize("collection_state", ["missing", "incompatible"]) +def test_registry_rendered_session_and_maintenance_configs_require_existing_qdrant_collection( + tmp_path, monkeypatch, collection_state, +): + session_path, maintenance_path = _render_registry_runtime_configs(tmp_path) + session_cfg = load_config(session_path) + maintenance_cfg = load_config(maintenance_path) - cfg = load_config(path) + assert session_cfg.vectors.collection_lifecycle == "require_existing" + assert maintenance_cfg.vectors.collection_lifecycle == "require_existing" + assert yaml.safe_load(session_path.read_text())["vectors"]["collection_lifecycle"] == "require_existing" + assert yaml.safe_load(maintenance_path.read_text())["vectors"]["collection_lifecycle"] == "require_existing" + assert session_path.read_bytes() == maintenance_path.read_bytes() - assert cfg.vectors.collection_lifecycle == "require_existing" + from test_qdrant_vector_store import FakeQdrantHttp, _write_record + + for cfg in (session_cfg, maintenance_cfg): + fake = FakeQdrantHttp(dimension=384) if collection_state == "incompatible" else FakeQdrantHttp() + if collection_state == "incompatible": + fake.collection = {"vectors": {"size": 384, "distance": "Cosine"}} + monkeypatch.setattr("requests.request", fake.request) + store = build_vector_store(cfg, require_write=True) + with pytest.raises(VectorStoreError, match="semantic_index_incompatible"): + store.upsert("memory", [_write_record("memory:1", "memory")]) + assert not [call for call in fake.calls if call[0] == "PUT"] + + +def test_registry_rendered_session_and_maintenance_configs_bind_equally(tmp_path): + session_path, maintenance_path = _render_registry_runtime_configs(tmp_path) + + assert config_dwh_binding(load_config(session_path)) == config_dwh_binding( + load_config(maintenance_path) + ) def test_legacy_runtime_config_keeps_create_capable_qdrant_default(tmp_path):