fix: serialize registry qdrant lifecycle
This commit is contained in:
@@ -255,6 +255,12 @@ export function renderRuntimeConfig(
|
|||||||
...(installation.profile === undefined ? {} : { profile: installation.profile }),
|
...(installation.profile === undefined ? {} : { profile: installation.profile }),
|
||||||
language: descriptor.workspace.language,
|
language: descriptor.workspace.language,
|
||||||
database,
|
database,
|
||||||
|
vectors: {
|
||||||
|
type: "qdrant",
|
||||||
|
base_url: semanticRuntime.internalQdrantUrl,
|
||||||
|
collection: descriptor.semantic_index.vector_store.collection,
|
||||||
|
collection_lifecycle: identity ? "require_existing" : "create_if_missing",
|
||||||
|
},
|
||||||
resources: {
|
resources: {
|
||||||
vector: {
|
vector: {
|
||||||
engine: "qdrant",
|
engine: "qdrant",
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { execFileSync } from "node:child_process";
|
||||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
@@ -80,6 +81,10 @@ test("renders only the schema-v3 internal Qdrant and Ollama runtime shape", () =
|
|||||||
ssl_ca_file: "/run/secrets/dwh-ca.pem", transport: "direct",
|
ssl_ca_file: "/run/secrets/dwh-ca.pem", transport: "direct",
|
||||||
},
|
},
|
||||||
dwh: { type: "postgres_direct" },
|
dwh: { type: "postgres_direct" },
|
||||||
|
vectors: {
|
||||||
|
type: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical",
|
||||||
|
collection_lifecycle: "require_existing",
|
||||||
|
},
|
||||||
resources: {
|
resources: {
|
||||||
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
|
vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" },
|
||||||
embeddings: {
|
embeddings: {
|
||||||
@@ -94,6 +99,63 @@ test("renders only the schema-v3 internal Qdrant and Ollama runtime shape", () =
|
|||||||
expect(rendered).not.toHaveProperty("vector_rest");
|
expect(rendered).not.toHaveProperty("vector_rest");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("keeps create_if_missing for non-registry runtime renders", () => {
|
||||||
|
const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, undefined, {}, semanticRuntime));
|
||||||
|
|
||||||
|
expect(rendered.vectors).toMatchObject({
|
||||||
|
type: "qdrant",
|
||||||
|
collection_lifecycle: "create_if_missing",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test.each(["session", "maintenance"])(
|
||||||
|
"renders require_existing vectors for %s acquisition",
|
||||||
|
(mode) => {
|
||||||
|
const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, {
|
||||||
|
workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40),
|
||||||
|
}, {}, semanticRuntime));
|
||||||
|
|
||||||
|
expect(rendered.vectors).toEqual({
|
||||||
|
type: "qdrant",
|
||||||
|
base_url: "http://qdrant:6333",
|
||||||
|
collection: "psd-clinical",
|
||||||
|
collection_lifecycle: "require_existing",
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test("session and maintenance renders are byte-identical and bind identically in harness", () => {
|
||||||
|
const context = { workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40) };
|
||||||
|
const passwordFile = evidenceSecretFile("dwh-password", "not-a-canary");
|
||||||
|
const bindings = {
|
||||||
|
...directBindings,
|
||||||
|
dwh: { ...directBindings.dwh, values: {
|
||||||
|
...directBindings.dwh.values,
|
||||||
|
THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: passwordFile,
|
||||||
|
} },
|
||||||
|
};
|
||||||
|
const session = renderRuntimeConfig(workspaceV3, bindings, paths, context, {}, semanticRuntime);
|
||||||
|
const maintenance = renderRuntimeConfig(workspaceV3, bindings, paths, context, {}, semanticRuntime);
|
||||||
|
|
||||||
|
expect(maintenance).toBe(session);
|
||||||
|
|
||||||
|
const script = `
|
||||||
|
import json, sys, tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from tht.config import load_config
|
||||||
|
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||||
|
with tempfile.TemporaryDirectory() as root:
|
||||||
|
path = Path(root) / "runtime.yaml"
|
||||||
|
path.write_text(sys.stdin.read())
|
||||||
|
print(json.dumps(config_dwh_binding(load_config(path)), sort_keys=True))
|
||||||
|
`;
|
||||||
|
const python = join(process.cwd(), "../harness/.venv/bin/python");
|
||||||
|
const run = (yaml: string) => execFileSync(python, ["-c", script], {
|
||||||
|
cwd: join(process.cwd(), "../harness"), input: yaml, encoding: "utf8",
|
||||||
|
}).trim();
|
||||||
|
expect(run(session)).toBe(run(maintenance));
|
||||||
|
});
|
||||||
|
|
||||||
test("renders schema-v3 DWH REST without exposing secret contents", () => {
|
test("renders schema-v3 DWH REST without exposing secret contents", () => {
|
||||||
const rendered = parse(renderRuntimeConfig(workspaceV3, {
|
const rendered = parse(renderRuntimeConfig(workspaceV3, {
|
||||||
dwh: {
|
dwh: {
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import json
|
import json
|
||||||
|
import subprocess
|
||||||
import traceback
|
import traceback
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import yaml
|
import yaml
|
||||||
@@ -7,10 +9,11 @@ from pydantic import SecretStr
|
|||||||
from typer.testing import CliRunner
|
from typer.testing import CliRunner
|
||||||
|
|
||||||
from tht.adapters.evidence import HttpManifestEvidenceSource
|
from tht.adapters.evidence import HttpManifestEvidenceSource
|
||||||
from tht.adapters.factory import build_evidence_sources
|
from tht.adapters.factory import build_evidence_sources, build_vector_store
|
||||||
from tht.cli import app
|
from tht.cli import app
|
||||||
from tht.config import ConfigError, load_config
|
from tht.config import ConfigError, load_config
|
||||||
from tht.jobs.dwh_pipeline import config_dwh_binding
|
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||||
|
from tht.ports.vector import VectorStoreError
|
||||||
|
|
||||||
SIGNED_CANARY = "SIGNED-CANARY-QUERY"
|
SIGNED_CANARY = "SIGNED-CANARY-QUERY"
|
||||||
ACCESS_CANARY = "ACCESS-CANARY"
|
ACCESS_CANARY = "ACCESS-CANARY"
|
||||||
@@ -353,6 +356,71 @@ def test_validation_repr_cli_and_exception_output_never_disclose_transport_secre
|
|||||||
assert_no_canaries(result.stderr)
|
assert_no_canaries(result.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def _render_registry_runtime_config(tmp_path) -> str:
|
||||||
|
"""Render the production schema-v3 runtime rather than duplicating its YAML."""
|
||||||
|
tmp_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
backend = Path(__file__).resolve().parents[2] / "backend"
|
||||||
|
password_file = tmp_path / "dwh-password"
|
||||||
|
password_file.write_text("not-a-canary")
|
||||||
|
script = r"""
|
||||||
|
import { parseWorkspaceYaml } from "__SCHEMA__";
|
||||||
|
import { renderRuntimeConfig } from "__RENDERER__";
|
||||||
|
const workspace = parseWorkspaceYaml(`workspace:
|
||||||
|
schema_version: 3
|
||||||
|
id: psd-clinical
|
||||||
|
name: Runtime test
|
||||||
|
language: en
|
||||||
|
dwh:
|
||||||
|
engine: postgres
|
||||||
|
database: analytics
|
||||||
|
schema: mart
|
||||||
|
supported_transports: [postgres_direct]
|
||||||
|
semantic_index:
|
||||||
|
vector_store:
|
||||||
|
engine: qdrant
|
||||||
|
collection: workspace-semantic
|
||||||
|
dimensions: 1024
|
||||||
|
distance: cosine
|
||||||
|
embedding:
|
||||||
|
provider: ollama_internal
|
||||||
|
model: qwen3-embedding:0.6b
|
||||||
|
dimensions: 1024
|
||||||
|
llm_policy:
|
||||||
|
allowed: [zai/glm-5.2]
|
||||||
|
`);
|
||||||
|
const bindings = {
|
||||||
|
dwh: { transport: "postgres_direct", missing: [], values: {
|
||||||
|
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal",
|
||||||
|
THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
|
||||||
|
THT_WS_PSD_CLINICAL_DWH_USER: "reader",
|
||||||
|
THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: process.argv[2],
|
||||||
|
} },
|
||||||
|
evidence: { missing: [], values: {} },
|
||||||
|
};
|
||||||
|
process.stdout.write(renderRuntimeConfig(workspace, bindings, {
|
||||||
|
sessions: "/tmp/sessions", artifacts: "/tmp/artifacts", indexes: "/tmp/indexes",
|
||||||
|
}, { workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40) }));
|
||||||
|
"""
|
||||||
|
script = script.replace(
|
||||||
|
"__SCHEMA__", str(backend / "src/workspaces/schema.ts"),
|
||||||
|
).replace("__RENDERER__", str(backend / "src/workspaces/runtime-renderer.ts"))
|
||||||
|
script_path = tmp_path / "render-runtime.mts"
|
||||||
|
script_path.write_text(script)
|
||||||
|
result = subprocess.run(
|
||||||
|
[str(backend / "node_modules/.bin/tsx"), str(script_path), str(password_file)],
|
||||||
|
cwd=backend, check=True, capture_output=True, text=True,
|
||||||
|
)
|
||||||
|
return result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def _render_registry_runtime_configs(tmp_path) -> tuple[Path, Path]:
|
||||||
|
session = tmp_path / "session.yaml"
|
||||||
|
maintenance = tmp_path / "maintenance.yaml"
|
||||||
|
session.write_text(_render_registry_runtime_config(tmp_path))
|
||||||
|
maintenance.write_text(_render_registry_runtime_config(tmp_path))
|
||||||
|
return session, maintenance
|
||||||
|
|
||||||
|
|
||||||
def _qdrant_config_yaml(tmp_path, *, registry: bool) -> dict:
|
def _qdrant_config_yaml(tmp_path, *, registry: bool) -> dict:
|
||||||
value = {
|
value = {
|
||||||
"dwh": {
|
"dwh": {
|
||||||
@@ -378,14 +446,39 @@ def _qdrant_config_yaml(tmp_path, *, registry: bool) -> dict:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("mode", ["session", "maintenance"])
|
@pytest.mark.parametrize("collection_state", ["missing", "incompatible"])
|
||||||
def test_registry_runtime_configs_require_existing_qdrant_collection(tmp_path, mode):
|
def test_registry_rendered_session_and_maintenance_configs_require_existing_qdrant_collection(
|
||||||
path = tmp_path / f"{mode}.yaml"
|
tmp_path, monkeypatch, collection_state,
|
||||||
path.write_text(yaml.safe_dump(_qdrant_config_yaml(tmp_path, registry=True)))
|
):
|
||||||
|
session_path, maintenance_path = _render_registry_runtime_configs(tmp_path)
|
||||||
|
session_cfg = load_config(session_path)
|
||||||
|
maintenance_cfg = load_config(maintenance_path)
|
||||||
|
|
||||||
cfg = load_config(path)
|
assert session_cfg.vectors.collection_lifecycle == "require_existing"
|
||||||
|
assert maintenance_cfg.vectors.collection_lifecycle == "require_existing"
|
||||||
|
assert yaml.safe_load(session_path.read_text())["vectors"]["collection_lifecycle"] == "require_existing"
|
||||||
|
assert yaml.safe_load(maintenance_path.read_text())["vectors"]["collection_lifecycle"] == "require_existing"
|
||||||
|
assert session_path.read_bytes() == maintenance_path.read_bytes()
|
||||||
|
|
||||||
assert cfg.vectors.collection_lifecycle == "require_existing"
|
from test_qdrant_vector_store import FakeQdrantHttp, _write_record
|
||||||
|
|
||||||
|
for cfg in (session_cfg, maintenance_cfg):
|
||||||
|
fake = FakeQdrantHttp(dimension=384) if collection_state == "incompatible" else FakeQdrantHttp()
|
||||||
|
if collection_state == "incompatible":
|
||||||
|
fake.collection = {"vectors": {"size": 384, "distance": "Cosine"}}
|
||||||
|
monkeypatch.setattr("requests.request", fake.request)
|
||||||
|
store = build_vector_store(cfg, require_write=True)
|
||||||
|
with pytest.raises(VectorStoreError, match="semantic_index_incompatible"):
|
||||||
|
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||||
|
assert not [call for call in fake.calls if call[0] == "PUT"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_rendered_session_and_maintenance_configs_bind_equally(tmp_path):
|
||||||
|
session_path, maintenance_path = _render_registry_runtime_configs(tmp_path)
|
||||||
|
|
||||||
|
assert config_dwh_binding(load_config(session_path)) == config_dwh_binding(
|
||||||
|
load_config(maintenance_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_runtime_config_keeps_create_capable_qdrant_default(tmp_path):
|
def test_legacy_runtime_config_keeps_create_capable_qdrant_default(tmp_path):
|
||||||
|
|||||||
Reference in New Issue
Block a user