From 8240fa4472bcbef90c4544b24aaefe364e223809 Mon Sep 17 00:00:00 2001 From: mptyl Date: Sat, 8 Aug 2026 20:29:47 +0200 Subject: [PATCH] test: add internal semantic smoke --- scripts/internal-semantic-smoke.sh | 298 ++++++++++++++++++++++++ scripts/task13-runtime-fixture-check.ts | 62 +++++ scripts/test-task13-runtime-fixtures.sh | 41 +++- scripts/unified-deployment-smoke.sh | 43 +++- 4 files changed, 441 insertions(+), 3 deletions(-) create mode 100755 scripts/internal-semantic-smoke.sh diff --git a/scripts/internal-semantic-smoke.sh b/scripts/internal-semantic-smoke.sh new file mode 100755 index 00000000..20e581d7 --- /dev/null +++ b/scripts/internal-semantic-smoke.sh @@ -0,0 +1,298 @@ +#!/usr/bin/env bash +# Live smoke for the mandatory internal semantic stack: disposable project/volumes, no host +# ports on semantic services, exact cleanup via Task 13 labels only, and offline persistence. +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd -P)" +# shellcheck source=./unified-deployment-smoke.sh +source "$root/scripts/unified-deployment-smoke.sh" + +task13_wait_internal_embedding_model() { + printf '== Wait for the internal embedding model ==\n' + for _attempt in $(seq 1 30); do + if task13_compose exec -T core /opt/venv/bin/python - <<'PY' >>"$TASK13_LOG" 2>&1 +import json +import urllib.request +import urllib.error + +with urllib.request.urlopen("http://embedding:11434/api/tags", timeout=10) as response: + payload = json.load(response) +models = [entry.get("name") for entry in payload.get("models", []) if isinstance(entry, dict)] +if "qwen3-embedding:0.6b" not in models: + raise SystemExit(1) +request = urllib.request.Request( + "http://embedding:11434/api/embed", + data=json.dumps({"model": "qwen3-embedding:0.6b", "input": ["warm semantic smoke"]}).encode("utf-8"), + headers={"content-type": "application/json"}, + method="POST", +) +try: + with urllib.request.urlopen(request, timeout=180) as response: + payload = json.load(response) +except urllib.error.URLError: + raise SystemExit(1) +embeddings = payload.get("embeddings") +raise SystemExit(0 if isinstance(embeddings, list) and len(embeddings) == 1 and len(embeddings[0]) == 1024 else 1) +PY + then + return 0 + fi + sleep 1 + done + task13_log_failure "internal embedding model readiness" +} + +task13_semantic_python_probe() { + local mode="$1" + task13_compose exec -T core /opt/venv/bin/python - "$mode" <<'PY' +from __future__ import annotations + +import hashlib +import json +import sys +from types import SimpleNamespace + +import requests + +from tht.adapters.vector.qdrant import QdrantVectorStore +from tht.ports.vector import VectorWriteRecord +from tht.vectorstore.embeddings import OllamaEmbeddings +from tht.vectorstore.records import VectorRecord + +MODE = sys.argv[1] +WORKSPACE_ID = "task13-smoke" +WORKSPACE_REVISION = "b" * 40 +COLLECTION = "task13-smoke" +QDRANT = "http://qdrant:6333" +EMBEDDING = "http://embedding:11434" +MODEL = "qwen3-embedding:0.6b" +DIM = 1024 + + +def embedder() -> OllamaEmbeddings: + return OllamaEmbeddings( + SimpleNamespace( + base_url=EMBEDDING, + model=MODEL, + dim=DIM, + connect_timeout=2.0, + timeout=180.0, + batch_size=8, + ) + ) + + +def store() -> QdrantVectorStore: + return QdrantVectorStore( + base_url=QDRANT, + collection=COLLECTION, + workspace_id=WORKSPACE_ID, + workspace_revision=WORKSPACE_REVISION, + expected_dimension=DIM, + ) + + +def content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def query_payload(vector: list[float], must: list[dict]) -> dict: + response = requests.post( + f"{QDRANT}/collections/{COLLECTION}/points/query", + json={ + "vector": vector, + "limit": 1, + "with_payload": True, + "filter": {"must": must}, + }, + timeout=(2.0, 15.0), + ) + response.raise_for_status() + payload = response.json() + points = payload.get("result", {}).get("points") + if not isinstance(points, list) or len(points) != 1: + raise RuntimeError(f"expected exactly one semantic point, got {payload!r}") + point = points[0] + result = point.get("payload") + if not isinstance(result, dict): + raise RuntimeError(f"missing payload in query result: {point!r}") + return result + + +records = { + "schema": { + "collection": "schema_records", + "record": VectorRecord( + id="schema_table:fact_task13", + kind="schema_table", + ref="fact_task13", + title="fact_task13", + content="Tabella fact_task13 con una riga dedicata allo smoke semantico interno.", + metadata={"table_name": "fact_task13"}, + ), + "query": "fact task13 smoke table", + "must": [ + {"key": "workspace_id", "match": {"value": WORKSPACE_ID}}, + {"key": "kind", "match": {"value": "schema"}}, + {"key": "record_kind", "match": {"value": "schema_table"}}, + {"key": "record_key", "match": {"value": "schema_table:fact_task13"}}, + ], + }, + "evidence": { + "collection": "evidence", + "record": VectorRecord( + id="evidence:task13-doc:0", + kind="evidence", + ref="task13-doc", + title="Task 13 Evidence", + content="Evidence dedicata allo smoke semantico interno con filtro esatto per generazione.", + metadata={ + "document_id": "task13-doc", + "vector_generation": "gen:11111111111111111111111111111111", + "status": "published", + "tier": "gold", + "tables": ["fact_task13"], + "concepts": ["semantic smoke"], + }, + ), + "query": "semantic smoke evidence generation", + "must": [ + {"key": "workspace_id", "match": {"value": WORKSPACE_ID}}, + {"key": "kind", "match": {"value": "evidence"}}, + {"key": "document_id", "match": {"value": "task13-doc"}}, + {"key": "vector_generation", "match": {"value": "gen:11111111111111111111111111111111"}}, + ], + }, + "memory": { + "collection": "memory", + "record": VectorRecord( + id="mem-9000", + kind="memory", + ref="mem-9000", + title="Task 13 Memory", + content="Memoria riusabile per lo smoke semantico interno persistente.", + metadata={ + "session_id": "task13-session", + "decision_seq": 9, + "subject": "semantic smoke memory", + "type": "concept_clarified", + "concepts": ["semantic smoke memory"], + }, + ), + "query": "semantic smoke memory reusable", + "must": [ + {"key": "workspace_id", "match": {"value": WORKSPACE_ID}}, + {"key": "kind", "match": {"value": "memory"}}, + {"key": "record_kind", "match": {"value": "memory"}}, + {"key": "record_key", "match": {"value": "mem-9000"}}, + ], + }, +} + +embedding_client = embedder() +vector_store = store() + +if MODE == "seed": + for family in records.values(): + record = family["record"] + vector_store.upsert( + family["collection"], + [ + VectorWriteRecord( + record=record, + embedding=embedding_client.embed_query(record.content), + content_hash=content_hash(record.content), + ) + ], + ) + +collection_info = requests.get( + f"{QDRANT}/collections/{COLLECTION}", + timeout=(2.0, 15.0), +) +collection_info.raise_for_status() +payload = collection_info.json() +size = payload.get("result", {}).get("config", {}).get("params", {}).get("vectors", {}).get("size") +distance = payload.get("result", {}).get("config", {}).get("params", {}).get("vectors", {}).get("distance") +if size != DIM or distance != "Cosine": + raise RuntimeError(f"unexpected Qdrant collection shape: size={size!r} distance={distance!r}") + +verified: dict[str, dict[str, str]] = {} +for family_name, family in records.items(): + payload = query_payload( + embedding_client.embed_query(family["query"]), + family["must"], + ) + if payload.get("workspace_id") != WORKSPACE_ID: + raise RuntimeError(f"{family_name} query leaked another workspace") + if payload.get("record_key") != family["record"].id: + raise RuntimeError(f"{family_name} query returned the wrong record key: {payload!r}") + verified[family_name] = { + "record_key": payload["record_key"], + "kind": payload["kind"], + } + +tags = requests.get(f"{EMBEDDING}/api/tags", timeout=(2.0, 15.0)) +tags.raise_for_status() +models = [entry.get("name") for entry in tags.json().get("models", []) if isinstance(entry, dict)] +if MODEL not in models: + raise RuntimeError(f"missing cached embedding model {MODEL}") + +print(json.dumps({ + "mode": MODE, + "collection": COLLECTION, + "model": MODEL, + "verified": verified, +}, sort_keys=True)) +PY +} + +task13_semantic_seed_and_assert() { + local output + printf '== Ensure the semantic collection and seed schema/evidence/memory ==\n' + output="$(task13_semantic_python_probe seed)" + printf '%s\n' "$output" >>"$TASK13_LOG" + grep -Fq '"schema"' <<<"$output" || task13_fail "schema semantic verification did not run" + grep -Fq '"evidence"' <<<"$output" || task13_fail "evidence semantic verification did not run" + grep -Fq '"memory"' <<<"$output" || task13_fail "memory semantic verification did not run" +} + +task13_semantic_verify_persistence() { + local output + printf '== Restart offline and prove semantic points plus model cache persist ==\n' + task13_write_environment /fixtures/offline.git + task13_compose_logged "offline semantic recreation" up --detach --force-recreate --wait --wait-timeout 120 + task13_wait_internal_embedding_model + task13_registry_status >>"$TASK13_LOG" 2>&1 || true + output="$(task13_semantic_python_probe verify)" + printf '%s\n' "$output" >>"$TASK13_LOG" + grep -Fq '"schema"' <<<"$output" || task13_fail "schema semantic persistence did not verify" + grep -Fq '"evidence"' <<<"$output" || task13_fail "evidence semantic persistence did not verify" + grep -Fq '"memory"' <<<"$output" || task13_fail "memory semantic persistence did not verify" +} + +task13_internal_semantic_smoke_main() { + local qdrant_before embedding_before + task13_initialize + task13_require_tools + task13_write_fixture_files + task13_write_environment /fixtures/remote.git + task13_seed_registry + task13_start_stack + task13_assert_project_ownership + task13_assert_built_image_ownership + task13_wait_internal_embedding_model + qdrant_before="$(task13_service_mount_fingerprint qdrant)" + embedding_before="$(task13_service_mount_fingerprint embedding)" + task13_semantic_seed_and_assert + task13_semantic_verify_persistence + [[ "$(task13_service_mount_fingerprint qdrant)" == "$qdrant_before" ]] \ + || task13_fail "offline recreation changed qdrant volume identity" + [[ "$(task13_service_mount_fingerprint embedding)" == "$embedding_before" ]] \ + || task13_fail "offline recreation changed embedding model cache volume identity" + printf 'Task 13 internal semantic smoke passed.\n' +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + task13_supervise "$TASK13_SMOKE_TIMEOUT" "internal semantic smoke" task13_internal_semantic_smoke_main +fi diff --git a/scripts/task13-runtime-fixture-check.ts b/scripts/task13-runtime-fixture-check.ts index 877b334e..6e6a2b6d 100644 --- a/scripts/task13-runtime-fixture-check.ts +++ b/scripts/task13-runtime-fixture-check.ts @@ -17,6 +17,25 @@ const workspace = parse(readFileSync(workspacePath, "utf8")); const core = config.services?.core; const frontend = config.services?.frontend; if (!core || !frontend) throw new Error("fixture render must contain core and frontend"); +const qdrant = config.services?.qdrant; +const embedding = config.services?.embedding; +const modelInit = config.services?.["embedding-model-init"]; +if (!qdrant || !embedding || !modelInit) { + throw new Error("fixture render must contain the private semantic services"); +} + +for (const [name, service, expectedExpose] of [ + ["qdrant", qdrant, "6333"], + ["embedding", embedding, "11434"], +] as const) { + if ((service.ports || []).length !== 0) throw new Error(`${name} must not publish host ports`); + if ((service.expose || []).join(",") !== expectedExpose) { + throw new Error(`${name} must expose only ${expectedExpose}`); + } +} +if ((modelInit.ports || []).length !== 0) { + throw new Error("embedding-model-init must not publish host ports"); +} const expected = { THT_WS_TASK13_SMOKE_DWH_TRANSPORT: "postgres_direct", @@ -34,6 +53,49 @@ for (const [name, value] of Object.entries(expected)) { } } +const semanticRuntime = { + THT_INTERNAL_QDRANT_URL: "http://qdrant:6333", + THT_INTERNAL_EMBEDDING_URL: "http://embedding:11434", + THT_INTERNAL_EMBEDDING_MODEL: "qwen3-embedding:0.6b", + THT_INTERNAL_EMBEDDING_DIMENSIONS: "1024", +}; +for (const [name, value] of Object.entries(semanticRuntime)) { + if (core.environment?.[name] !== value) { + throw new Error(`core semantic runtime ${name} is ${JSON.stringify(core.environment?.[name])}, want ${JSON.stringify(value)}`); + } + if (Object.hasOwn(frontend.environment || {}, name)) { + throw new Error(`semantic runtime escaped to frontend: ${name}`); + } +} +for (const name of ["THT_VEC_REST_URL", "THT_VEC_WRITE_REST_URL", "THT_OLLAMA_URL"]) { + if (Object.hasOwn(core.environment || {}, name) && core.environment?.[name] !== "") { + throw new Error(`fixture render reintroduced external semantic binding ${name}`); + } +} + +if (workspace.workspace?.id !== "task13-smoke") throw new Error("fixture workspace id changed"); +if (workspace.semantic_index?.vector_store?.engine !== "qdrant") { + throw new Error("fixture workspace must use qdrant"); +} +if (workspace.semantic_index?.vector_store?.collection !== workspace.workspace?.id) { + throw new Error("fixture workspace must dedicate one qdrant collection per workspace id"); +} +if (workspace.semantic_index?.vector_store?.dimensions !== 1024) { + throw new Error("fixture workspace qdrant dimension changed"); +} +if (workspace.semantic_index?.vector_store?.distance !== "cosine") { + throw new Error("fixture workspace qdrant distance changed"); +} +if (workspace.semantic_index?.embedding?.provider !== "ollama_internal") { + throw new Error("fixture workspace must use internal ollama embeddings"); +} +if (workspace.semantic_index?.embedding?.model !== "qwen3-embedding:0.6b") { + throw new Error("fixture workspace embedding model changed"); +} +if (workspace.semantic_index?.embedding?.dimensions !== 1024) { + throw new Error("fixture workspace embedding dimension changed"); +} + const bundle = config.secrets?.thothii_secrets; const bundleSource = bundle?.file; if (typeof bundleSource !== "string" || !statSync(bundleSource).isFile()) { diff --git a/scripts/test-task13-runtime-fixtures.sh b/scripts/test-task13-runtime-fixtures.sh index c7c4b476..fb7c7a00 100755 --- a/scripts/test-task13-runtime-fixtures.sh +++ b/scripts/test-task13-runtime-fixtures.sh @@ -102,7 +102,13 @@ checker=(node --import "$tsx_loader" "$root/scripts/task13-runtime-fixture-check } "${checker[@]}" "$rendered" "$workspace" "$profile" -for mutation in wrong-service wrong-value wrong-secret-mount; do +for mutation in \ + wrong-service \ + wrong-value \ + wrong-secret-mount \ + wrong-qdrant-service \ + wrong-embedding-service \ + external-semantic-urls; do mutated="$fixture/$mutation.json" node - "$rendered" "$mutated" "$mutation" <<'NODE' const fs = require("fs"); @@ -115,8 +121,15 @@ if (mutation === "wrong-service") { delete config.services.core.environment[name]; } else if (mutation === "wrong-value") { config.services.core.environment.THT_WS_TASK13_SMOKE_DWH_HOST = "wrong.task13.invalid"; -} else { +} else if (mutation === "wrong-secret-mount") { config.secrets.thothii_secrets.file = source + ".missing"; +} else if (mutation === "wrong-qdrant-service") { + config.services.core.environment.THT_INTERNAL_QDRANT_URL = "http://vector:6333"; +} else if (mutation === "wrong-embedding-service") { + config.services.core.environment.THT_INTERNAL_EMBEDDING_URL = "http://ollama:11434"; +} else if (mutation === "external-semantic-urls") { + config.services.core.environment.THT_INTERNAL_QDRANT_URL = "https://qdrant.example.test"; + config.services.core.environment.THT_INTERNAL_EMBEDDING_URL = "https://embedding.example.test"; } fs.writeFileSync(destination, JSON.stringify(config)); NODE @@ -127,4 +140,28 @@ NODE fi done +for mutation in collection-reuse dimension-change; do + mutated="$fixture/$mutation.yaml" + node - "$root/backend/package.json" "$workspace" "$mutated" "$mutation" <<'NODE' +const fs = require("fs"); +const { createRequire } = require("module"); +const requireFromBackend = createRequire(process.argv[2]); +const yaml = requireFromBackend("yaml"); +const [source, destination, mutation] = process.argv.slice(3); +const workspace = yaml.parse(fs.readFileSync(source, "utf8")); +if (mutation === "collection-reuse") { + workspace.semantic_index.vector_store.collection = "shared-semantic"; +} else if (mutation === "dimension-change") { + workspace.semantic_index.vector_store.dimensions = 1536; + workspace.semantic_index.embedding.dimensions = 1536; +} +fs.writeFileSync(destination, yaml.stringify(workspace)); +NODE + if "${checker[@]}" "$rendered" "$mutated" "$profile" \ + >"$fixture/$mutation.out" 2>"$fixture/$mutation.err"; then + echo "runtime fixture checker accepted workspace mutation: $mutation" >&2 + exit 1 + fi +done + echo "Task 13 $profile rendered runtime fixture contract passed." diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index f7c34c24..96185b09 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -304,6 +304,15 @@ services: io.thothii.task13.run: "$TASK13_RUN_ID" labels: io.thothii.task13.run: "$TASK13_RUN_ID" + qdrant: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" + embedding: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" + embedding-model-init: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" networks: thothii: labels: @@ -321,6 +330,12 @@ volumes: sessions: labels: io.thothii.task13.run: "$TASK13_RUN_ID" + qdrant-data: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" + embedding-models: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" EOF chmod 0600 "$TASK13_OVERRIDE" @@ -405,12 +420,28 @@ services: io.thothii.task13.run: "$TASK13_RUN_ID" labels: io.thothii.task13.run: "$TASK13_RUN_ID" + qdrant: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" + embedding: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" + embedding-model-init: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" session-migrate: image: $TASK13_CORE_IMAGE networks: thothii: labels: io.thothii.task13.run: "$TASK13_RUN_ID" +volumes: + qdrant-data: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" + embedding-models: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" EOF chmod 0600 "$TASK13_OVERRIDE" @@ -744,6 +775,12 @@ task13_mount_fingerprint() { | LC_ALL=C sort } +task13_service_mount_fingerprint() { + local service="$1" + docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' \ + "$(task13_compose ps -q "$service")" | LC_ALL=C sort +} + task13_prepare_persistence() { task13_compose exec -T core sh -ceu ' printf %s settings-preserved > /data/settings/task13-settings @@ -1368,6 +1405,9 @@ task13_self_test_public_timeout_contract() { grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \ "$root/scripts/thothctl-update-smoke.sh" \ || task13_fail "direct update smoke invocation lacks an internal supervisor" + grep -Eq 'task13_supervise[[:space:]].*task13_internal_semantic_smoke_main' \ + "$root/scripts/internal-semantic-smoke.sh" \ + || task13_fail "direct internal semantic smoke invocation lacks an internal supervisor" } task13_self_test_windows_release_contract() { @@ -1428,7 +1468,8 @@ task13_self_test_source_contract() { registry_function='task13_start_''registry' if rg -n 'docker[[:space:]]+(system[[:space:]]+)?prune' \ "$root/scripts/unified-deployment-smoke.sh" \ - "$root/scripts/thothctl-update-smoke.sh" >/dev/null; then + "$root/scripts/thothctl-update-smoke.sh" \ + "$root/scripts/internal-semantic-smoke.sh" >/dev/null; then task13_fail "Task 13 smoke scripts must never prune global Docker state" fi ! grep -Fq -- "$host_network" "$root/scripts/unified-deployment-smoke.sh" \