#!/usr/bin/env bash # Live smoke for the mandatory internal semantic stack: disposable project/volumes, no host # ports on semantic services, exact cleanup via Task 13 labels only, and offline persistence. set -euo pipefail root="$(cd "$(dirname "$0")/.." && pwd -P)" # shellcheck source=./unified-deployment-smoke.sh source "$root/scripts/unified-deployment-smoke.sh" task13_write_offline_semantic_override() { TASK13_OFFLINE_OVERRIDE="$TASK13_TMP/compose.task13.offline-semantic.yaml" cat >"$TASK13_OFFLINE_OVERRIDE" <<'EOF' networks: thothii: internal: true EOF chmod 0600 "$TASK13_OFFLINE_OVERRIDE" } task13_offline_semantic_compose() { docker compose \ --project-name "$TASK13_PROJECT" \ --project-directory "$TASK13_ROOT" \ --env-file "$TASK13_ENV_FILE" \ -f "$TASK13_ROOT/compose.yaml" \ -f "$TASK13_ROOT/deploy/compose.local.yaml" \ -f "$TASK13_OVERRIDE" \ -f "$TASK13_OFFLINE_OVERRIDE" \ "$@" } task13_offline_semantic_compose_logged() { local label="$1" shift task13_run_logged "$label" task13_offline_semantic_compose "$@" } task13_wait_internal_embedding_model() { printf '== Wait for the internal embedding model ==\n' for _attempt in $(seq 1 30); do if task13_compose exec -T core /opt/venv/bin/python - <<'PY' >>"$TASK13_LOG" 2>&1 import json import urllib.request import urllib.error with urllib.request.urlopen("http://embedding:11434/api/tags", timeout=10) as response: payload = json.load(response) models = [entry.get("name") for entry in payload.get("models", []) if isinstance(entry, dict)] if "qwen3-embedding:0.6b" not in models: raise SystemExit(1) request = urllib.request.Request( "http://embedding:11434/api/embed", data=json.dumps({"model": "qwen3-embedding:0.6b", "input": ["warm semantic smoke"]}).encode("utf-8"), headers={"content-type": "application/json"}, method="POST", ) try: with urllib.request.urlopen(request, timeout=180) as response: payload = json.load(response) except urllib.error.URLError: raise SystemExit(1) embeddings = payload.get("embeddings") raise SystemExit(0 if isinstance(embeddings, list) and len(embeddings) == 1 and len(embeddings[0]) == 1024 else 1) PY then return 0 fi sleep 1 done task13_log_failure "internal embedding model readiness" } task13_wait_internal_embedding_model_offline() { printf '== Wait for the internal embedding model ==\n' for _attempt in $(seq 1 30); do if docker run --rm -i --pull never \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --network "$TASK13_NETWORK" \ --entrypoint /opt/venv/bin/python \ "$TASK13_CORE_IMAGE" - <<'PY' >>"$TASK13_LOG" 2>&1 import json import urllib.request import urllib.error with urllib.request.urlopen("http://embedding:11434/api/tags", timeout=10) as response: payload = json.load(response) models = [entry.get("name") for entry in payload.get("models", []) if isinstance(entry, dict)] if "qwen3-embedding:0.6b" not in models: raise SystemExit(1) request = urllib.request.Request( "http://embedding:11434/api/embed", data=json.dumps({"model": "qwen3-embedding:0.6b", "input": ["warm semantic smoke"]}).encode("utf-8"), headers={"content-type": "application/json"}, method="POST", ) try: with urllib.request.urlopen(request, timeout=180) as response: payload = json.load(response) except urllib.error.URLError: raise SystemExit(1) embeddings = payload.get("embeddings") raise SystemExit(0 if isinstance(embeddings, list) and len(embeddings) == 1 and len(embeddings[0]) == 1024 else 1) PY then return 0 fi sleep 1 done task13_log_failure "offline internal embedding model readiness" } task13_semantic_python_probe_with() { local mode="$1" shift "$@" "$mode" <<'PY' from __future__ import annotations import hashlib import json import sys from types import SimpleNamespace import requests from tht.adapters.vector.qdrant import QdrantVectorStore from tht.ports.vector import VectorWriteRecord from tht.vectorstore.embeddings import OllamaEmbeddings from tht.vectorstore.records import VectorRecord MODE = sys.argv[1] WORKSPACE_ID = "task13-smoke" WORKSPACE_REVISION = "b" * 40 COLLECTION = "task13-smoke" QDRANT = "http://qdrant:6333" EMBEDDING = "http://embedding:11434" MODEL = "qwen3-embedding:0.6b" DIM = 1024 def embedder() -> OllamaEmbeddings: return OllamaEmbeddings( SimpleNamespace( base_url=EMBEDDING, model=MODEL, dim=DIM, connect_timeout=2.0, timeout=180.0, batch_size=8, ) ) def store() -> QdrantVectorStore: return QdrantVectorStore( base_url=QDRANT, collection=COLLECTION, workspace_id=WORKSPACE_ID, workspace_revision=WORKSPACE_REVISION, expected_dimension=DIM, ) def content_hash(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() def query_payload(vector: list[float], must: list[dict]) -> dict: response = requests.post( f"{QDRANT}/collections/{COLLECTION}/points/query", json={ "vector": vector, "limit": 1, "with_payload": True, "filter": {"must": must}, }, timeout=(2.0, 15.0), ) response.raise_for_status() payload = response.json() points = payload.get("result", {}).get("points") if not isinstance(points, list) or len(points) != 1: raise RuntimeError(f"expected exactly one semantic point, got {payload!r}") point = points[0] result = point.get("payload") if not isinstance(result, dict): raise RuntimeError(f"missing payload in query result: {point!r}") return result records = { "schema": { "collection": "schema_records", "record": VectorRecord( id="schema_table:fact_task13", kind="schema_table", ref="fact_task13", title="fact_task13", content="Tabella fact_task13 con una riga dedicata allo smoke semantico interno.", metadata={"table_name": "fact_task13"}, ), "query": "fact task13 smoke table", "must": [ {"key": "workspace_id", "match": {"value": WORKSPACE_ID}}, {"key": "kind", "match": {"value": "schema"}}, {"key": "record_kind", "match": {"value": "schema_table"}}, {"key": "record_key", "match": {"value": "schema_table:fact_task13"}}, ], }, "evidence": { "collection": "evidence", "record": VectorRecord( id="evidence:task13-doc:0", kind="evidence", ref="task13-doc", title="Task 13 Evidence", content="Evidence dedicata allo smoke semantico interno con filtro esatto per generazione.", metadata={ "document_id": "task13-doc", "vector_generation": "gen:11111111111111111111111111111111", "status": "published", "tier": "gold", "tables": ["fact_task13"], "concepts": ["semantic smoke"], }, ), "query": "semantic smoke evidence generation", "must": [ {"key": "workspace_id", "match": {"value": WORKSPACE_ID}}, {"key": "kind", "match": {"value": "evidence"}}, {"key": "document_id", "match": {"value": "task13-doc"}}, {"key": "vector_generation", "match": {"value": "gen:11111111111111111111111111111111"}}, ], }, "memory": { "collection": "memory", "record": VectorRecord( id="mem-9000", kind="memory", ref="mem-9000", title="Task 13 Memory", content="Memoria riusabile per lo smoke semantico interno persistente.", metadata={ "session_id": "task13-session", "decision_seq": 9, "subject": "semantic smoke memory", "type": "concept_clarified", "concepts": ["semantic smoke memory"], }, ), "query": "semantic smoke memory reusable", "must": [ {"key": "workspace_id", "match": {"value": WORKSPACE_ID}}, {"key": "kind", "match": {"value": "memory"}}, {"key": "record_kind", "match": {"value": "memory"}}, {"key": "record_key", "match": {"value": "mem-9000"}}, ], }, } embedding_client = embedder() vector_store = store() if MODE == "seed": for family in records.values(): record = family["record"] vector_store.upsert( family["collection"], [ VectorWriteRecord( record=record, embedding=embedding_client.embed_query(record.content), content_hash=content_hash(record.content), ) ], ) collection_info = requests.get( f"{QDRANT}/collections/{COLLECTION}", timeout=(2.0, 15.0), ) collection_info.raise_for_status() payload = collection_info.json() size = payload.get("result", {}).get("config", {}).get("params", {}).get("vectors", {}).get("size") distance = payload.get("result", {}).get("config", {}).get("params", {}).get("vectors", {}).get("distance") if size != DIM or distance != "Cosine": raise RuntimeError(f"unexpected Qdrant collection shape: size={size!r} distance={distance!r}") verified: dict[str, dict[str, str]] = {} for family_name, family in records.items(): payload = query_payload( embedding_client.embed_query(family["query"]), family["must"], ) if payload.get("workspace_id") != WORKSPACE_ID: raise RuntimeError(f"{family_name} query leaked another workspace") if payload.get("record_key") != family["record"].id: raise RuntimeError(f"{family_name} query returned the wrong record key: {payload!r}") verified[family_name] = { "record_key": payload["record_key"], "kind": payload["kind"], } tags = requests.get(f"{EMBEDDING}/api/tags", timeout=(2.0, 15.0)) tags.raise_for_status() models = [entry.get("name") for entry in tags.json().get("models", []) if isinstance(entry, dict)] if MODEL not in models: raise RuntimeError(f"missing cached embedding model {MODEL}") print(json.dumps({ "mode": MODE, "collection": COLLECTION, "model": MODEL, "verified": verified, }, sort_keys=True)) PY } task13_semantic_python_probe() { local mode="$1" task13_semantic_python_probe_with "$mode" task13_compose exec -T core /opt/venv/bin/python - } task13_semantic_python_probe_offline() { local mode="$1" task13_semantic_python_probe_with "$mode" docker run --rm -i --pull never \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --network "$TASK13_NETWORK" \ --entrypoint /opt/venv/bin/python \ "$TASK13_CORE_IMAGE" - } task13_semantic_seed_and_assert() { local output printf '== Ensure the semantic collection and seed schema/evidence/memory ==\n' output="$(task13_semantic_python_probe seed)" printf '%s\n' "$output" >>"$TASK13_LOG" grep -Fq '"schema"' <<<"$output" || task13_fail "schema semantic verification did not run" grep -Fq '"evidence"' <<<"$output" || task13_fail "evidence semantic verification did not run" grep -Fq '"memory"' <<<"$output" || task13_fail "memory semantic verification did not run" } task13_assert_offline_semantic_isolation() { local running [[ "$(docker network inspect --format '{{.Internal}}' "$TASK13_NETWORK")" == true ]] \ || task13_fail "offline semantic network still allows egress" running="$(task13_offline_semantic_compose ps --services --status running | sort)" [[ "$running" == $'embedding\nqdrant' ]] \ || task13_fail "offline semantic recreation started non-semantic services" for service in core frontend embedding-model-init; do if docker ps -a --filter "label=com.docker.compose.project=$TASK13_PROJECT" \ --filter "label=com.docker.compose.service=$service" --format '{{.ID}}' | grep -q .; then task13_fail "offline semantic recreation invoked bootstrap service $service" fi done } task13_semantic_verify_persistence() { local output printf '== Restart offline and prove semantic points plus model cache persist ==\n' task13_write_environment /fixtures/offline.git task13_write_offline_semantic_override task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" >>"$TASK13_LOG" 2>&1 \ || task13_fail "offline semantic phase could not remove the temporary LLM fixture" task13_compose_logged "offline semantic stop" down --remove-orphans --timeout 10 task13_offline_semantic_compose_logged "offline semantic recreation" \ up --detach --wait --wait-timeout 120 --pull never qdrant embedding TASK13_NETWORK="$(docker network ls \ --filter "label=com.docker.compose.project=$TASK13_PROJECT" \ --filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')" [[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] || task13_fail "offline semantic network was not resolved" task13_assert_offline_semantic_isolation task13_wait_internal_embedding_model_offline output="$(task13_semantic_python_probe_offline verify)" printf '%s\n' "$output" >>"$TASK13_LOG" grep -Fq '"schema"' <<<"$output" || task13_fail "schema semantic persistence did not verify" grep -Fq '"evidence"' <<<"$output" || task13_fail "evidence semantic persistence did not verify" grep -Fq '"memory"' <<<"$output" || task13_fail "memory semantic persistence did not verify" } task13_internal_semantic_smoke_main() { local qdrant_before embedding_before task13_initialize task13_require_tools task13_write_fixture_files task13_write_environment /fixtures/remote.git task13_seed_registry task13_start_stack task13_assert_project_ownership task13_assert_built_image_ownership task13_wait_internal_embedding_model qdrant_before="$(task13_service_mount_fingerprint qdrant)" embedding_before="$(task13_service_mount_fingerprint embedding)" task13_semantic_seed_and_assert task13_semantic_verify_persistence [[ "$(task13_service_mount_fingerprint qdrant)" == "$qdrant_before" ]] \ || task13_fail "offline recreation changed qdrant volume identity" [[ "$(task13_service_mount_fingerprint embedding)" == "$embedding_before" ]] \ || task13_fail "offline recreation changed embedding model cache volume identity" printf 'Task 13 internal semantic smoke passed.\n' } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then task13_supervise "$TASK13_SMOKE_TIMEOUT" "internal semantic smoke" task13_internal_semantic_smoke_main fi