fix: serialize registry qdrant lifecycle
This commit is contained in:
@@ -255,6 +255,12 @@ export function renderRuntimeConfig(
|
||||
...(installation.profile === undefined ? {} : { profile: installation.profile }),
|
||||
language: descriptor.workspace.language,
|
||||
database,
|
||||
vectors: {
|
||||
type: "qdrant",
|
||||
base_url: semanticRuntime.internalQdrantUrl,
|
||||
collection: descriptor.semantic_index.vector_store.collection,
|
||||
collection_lifecycle: identity ? "require_existing" : "create_if_missing",
|
||||
},
|
||||
resources: {
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -80,6 +81,10 @@ test("renders only the schema-v3 internal Qdrant and Ollama runtime shape", () =
|
||||
ssl_ca_file: "/run/secrets/dwh-ca.pem", transport: "direct",
|
||||
},
|
||||
dwh: { type: "postgres_direct" },
|
||||
vectors: {
|
||||
type: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical",
|
||||
collection_lifecycle: "require_existing",
|
||||
},
|
||||
resources: {
|
||||
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
|
||||
embeddings: {
|
||||
@@ -94,6 +99,63 @@ test("renders only the schema-v3 internal Qdrant and Ollama runtime shape", () =
|
||||
expect(rendered).not.toHaveProperty("vector_rest");
|
||||
});
|
||||
|
||||
test("keeps create_if_missing for non-registry runtime renders", () => {
|
||||
const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, undefined, {}, semanticRuntime));
|
||||
|
||||
expect(rendered.vectors).toMatchObject({
|
||||
type: "qdrant",
|
||||
collection_lifecycle: "create_if_missing",
|
||||
});
|
||||
});
|
||||
|
||||
test.each(["session", "maintenance"])(
|
||||
"renders require_existing vectors for %s acquisition",
|
||||
(mode) => {
|
||||
const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, {
|
||||
workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40),
|
||||
}, {}, semanticRuntime));
|
||||
|
||||
expect(rendered.vectors).toEqual({
|
||||
type: "qdrant",
|
||||
base_url: "http://qdrant:6333",
|
||||
collection: "psd-clinical",
|
||||
collection_lifecycle: "require_existing",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test("session and maintenance renders are byte-identical and bind identically in harness", () => {
|
||||
const context = { workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40) };
|
||||
const passwordFile = evidenceSecretFile("dwh-password", "not-a-canary");
|
||||
const bindings = {
|
||||
...directBindings,
|
||||
dwh: { ...directBindings.dwh, values: {
|
||||
...directBindings.dwh.values,
|
||||
THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: passwordFile,
|
||||
} },
|
||||
};
|
||||
const session = renderRuntimeConfig(workspaceV3, bindings, paths, context, {}, semanticRuntime);
|
||||
const maintenance = renderRuntimeConfig(workspaceV3, bindings, paths, context, {}, semanticRuntime);
|
||||
|
||||
expect(maintenance).toBe(session);
|
||||
|
||||
const script = `
|
||||
import json, sys, tempfile
|
||||
from pathlib import Path
|
||||
from tht.config import load_config
|
||||
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
path = Path(root) / "runtime.yaml"
|
||||
path.write_text(sys.stdin.read())
|
||||
print(json.dumps(config_dwh_binding(load_config(path)), sort_keys=True))
|
||||
`;
|
||||
const python = join(process.cwd(), "../harness/.venv/bin/python");
|
||||
const run = (yaml: string) => execFileSync(python, ["-c", script], {
|
||||
cwd: join(process.cwd(), "../harness"), input: yaml, encoding: "utf8",
|
||||
}).trim();
|
||||
expect(run(session)).toBe(run(maintenance));
|
||||
});
|
||||
|
||||
test("renders schema-v3 DWH REST without exposing secret contents", () => {
|
||||
const rendered = parse(renderRuntimeConfig(workspaceV3, {
|
||||
dwh: {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import json
|
||||
import subprocess
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
@@ -7,10 +9,11 @@ from pydantic import SecretStr
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from tht.adapters.evidence import HttpManifestEvidenceSource
|
||||
from tht.adapters.factory import build_evidence_sources
|
||||
from tht.adapters.factory import build_evidence_sources, build_vector_store
|
||||
from tht.cli import app
|
||||
from tht.config import ConfigError, load_config
|
||||
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||
from tht.ports.vector import VectorStoreError
|
||||
|
||||
SIGNED_CANARY = "SIGNED-CANARY-QUERY"
|
||||
ACCESS_CANARY = "ACCESS-CANARY"
|
||||
@@ -353,6 +356,71 @@ def test_validation_repr_cli_and_exception_output_never_disclose_transport_secre
|
||||
assert_no_canaries(result.stderr)
|
||||
|
||||
|
||||
def _render_registry_runtime_config(tmp_path) -> str:
|
||||
"""Render the production schema-v3 runtime rather than duplicating its YAML."""
|
||||
tmp_path.mkdir(parents=True, exist_ok=True)
|
||||
backend = Path(__file__).resolve().parents[2] / "backend"
|
||||
password_file = tmp_path / "dwh-password"
|
||||
password_file.write_text("not-a-canary")
|
||||
script = r"""
|
||||
import { parseWorkspaceYaml } from "__SCHEMA__";
|
||||
import { renderRuntimeConfig } from "__RENDERER__";
|
||||
const workspace = parseWorkspaceYaml(`workspace:
|
||||
schema_version: 3
|
||||
id: psd-clinical
|
||||
name: Runtime test
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: analytics
|
||||
schema: mart
|
||||
supported_transports: [postgres_direct]
|
||||
semantic_index:
|
||||
vector_store:
|
||||
engine: qdrant
|
||||
collection: workspace-semantic
|
||||
dimensions: 1024
|
||||
distance: cosine
|
||||
embedding:
|
||||
provider: ollama_internal
|
||||
model: qwen3-embedding:0.6b
|
||||
dimensions: 1024
|
||||
llm_policy:
|
||||
allowed: [zai/glm-5.2]
|
||||
`);
|
||||
const bindings = {
|
||||
dwh: { transport: "postgres_direct", missing: [], values: {
|
||||
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal",
|
||||
THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
|
||||
THT_WS_PSD_CLINICAL_DWH_USER: "reader",
|
||||
THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: process.argv[2],
|
||||
} },
|
||||
evidence: { missing: [], values: {} },
|
||||
};
|
||||
process.stdout.write(renderRuntimeConfig(workspace, bindings, {
|
||||
sessions: "/tmp/sessions", artifacts: "/tmp/artifacts", indexes: "/tmp/indexes",
|
||||
}, { workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40) }));
|
||||
"""
|
||||
script = script.replace(
|
||||
"__SCHEMA__", str(backend / "src/workspaces/schema.ts"),
|
||||
).replace("__RENDERER__", str(backend / "src/workspaces/runtime-renderer.ts"))
|
||||
script_path = tmp_path / "render-runtime.mts"
|
||||
script_path.write_text(script)
|
||||
result = subprocess.run(
|
||||
[str(backend / "node_modules/.bin/tsx"), str(script_path), str(password_file)],
|
||||
cwd=backend, check=True, capture_output=True, text=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _render_registry_runtime_configs(tmp_path) -> tuple[Path, Path]:
|
||||
session = tmp_path / "session.yaml"
|
||||
maintenance = tmp_path / "maintenance.yaml"
|
||||
session.write_text(_render_registry_runtime_config(tmp_path))
|
||||
maintenance.write_text(_render_registry_runtime_config(tmp_path))
|
||||
return session, maintenance
|
||||
|
||||
|
||||
def _qdrant_config_yaml(tmp_path, *, registry: bool) -> dict:
|
||||
value = {
|
||||
"dwh": {
|
||||
@@ -378,14 +446,39 @@ def _qdrant_config_yaml(tmp_path, *, registry: bool) -> dict:
|
||||
return value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["session", "maintenance"])
|
||||
def test_registry_runtime_configs_require_existing_qdrant_collection(tmp_path, mode):
|
||||
path = tmp_path / f"{mode}.yaml"
|
||||
path.write_text(yaml.safe_dump(_qdrant_config_yaml(tmp_path, registry=True)))
|
||||
@pytest.mark.parametrize("collection_state", ["missing", "incompatible"])
|
||||
def test_registry_rendered_session_and_maintenance_configs_require_existing_qdrant_collection(
|
||||
tmp_path, monkeypatch, collection_state,
|
||||
):
|
||||
session_path, maintenance_path = _render_registry_runtime_configs(tmp_path)
|
||||
session_cfg = load_config(session_path)
|
||||
maintenance_cfg = load_config(maintenance_path)
|
||||
|
||||
cfg = load_config(path)
|
||||
assert session_cfg.vectors.collection_lifecycle == "require_existing"
|
||||
assert maintenance_cfg.vectors.collection_lifecycle == "require_existing"
|
||||
assert yaml.safe_load(session_path.read_text())["vectors"]["collection_lifecycle"] == "require_existing"
|
||||
assert yaml.safe_load(maintenance_path.read_text())["vectors"]["collection_lifecycle"] == "require_existing"
|
||||
assert session_path.read_bytes() == maintenance_path.read_bytes()
|
||||
|
||||
assert cfg.vectors.collection_lifecycle == "require_existing"
|
||||
from test_qdrant_vector_store import FakeQdrantHttp, _write_record
|
||||
|
||||
for cfg in (session_cfg, maintenance_cfg):
|
||||
fake = FakeQdrantHttp(dimension=384) if collection_state == "incompatible" else FakeQdrantHttp()
|
||||
if collection_state == "incompatible":
|
||||
fake.collection = {"vectors": {"size": 384, "distance": "Cosine"}}
|
||||
monkeypatch.setattr("requests.request", fake.request)
|
||||
store = build_vector_store(cfg, require_write=True)
|
||||
with pytest.raises(VectorStoreError, match="semantic_index_incompatible"):
|
||||
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||
assert not [call for call in fake.calls if call[0] == "PUT"]
|
||||
|
||||
|
||||
def test_registry_rendered_session_and_maintenance_configs_bind_equally(tmp_path):
|
||||
session_path, maintenance_path = _render_registry_runtime_configs(tmp_path)
|
||||
|
||||
assert config_dwh_binding(load_config(session_path)) == config_dwh_binding(
|
||||
load_config(maintenance_path)
|
||||
)
|
||||
|
||||
|
||||
def test_legacy_runtime_config_keeps_create_capable_qdrant_default(tmp_path):
|
||||
|
||||
Reference in New Issue
Block a user