feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s

This commit is contained in:
Codex
2026-09-06 17:49:35 +02:00
parent 8707ae1d46
commit cffa60772e
141 changed files with 5898 additions and 3015 deletions
@@ -0,0 +1,103 @@
import json
from types import SimpleNamespace
from tht.mschema.catalog_snapshot import load_catalog_metadata_snapshot
from tht.mschema.context import load_schema_context
from tht.search import SearchResult, schema_tables
from tht.vectorstore.records import catalog_schema_records, qdrant_semantic_kind
def _snapshot():
return {
"schemaVersion": 1,
"workspaceId": "sales",
"databaseId": "db-1",
"databaseName": "warehouse",
"schemaName": "analytics",
"metadataContentRevision": 7,
"tables": [
{
"id": "table-orders",
"name": "orders",
"description": "Customer orders",
"descriptionSource": "curated",
"columns": [
{
"id": "column-customer",
"name": "customer_id",
"ordinalPosition": 1,
"dataType": "uuid",
"isNullable": False,
"defaultExpression": None,
"primaryKeyPosition": None,
"sensitive": False,
"description": "Ordering customer",
"descriptionSource": "generated",
}
],
},
{
"id": "table-customers",
"name": "customers",
"description": None,
"descriptionSource": None,
"columns": [
{
"id": "column-id",
"name": "id",
"ordinalPosition": 1,
"dataType": "uuid",
"isNullable": False,
"defaultExpression": None,
"primaryKeyPosition": 1,
"sensitive": True,
"description": "Customer identifier",
"descriptionSource": "source_comment",
}
],
},
],
"relationships": [
{
"id": "relationship-1",
"origin": "physical",
"sourceTable": "orders",
"sourceColumns": ["customer_id"],
"targetTable": "customers",
"targetColumns": ["id"],
}
],
}
def test_catalog_snapshot_is_the_schema_context_and_vector_source(tmp_path):
path = tmp_path / "catalog-metadata.json"
path.write_text(json.dumps(_snapshot()))
cfg = SimpleNamespace(
paths=SimpleNamespace(catalog_metadata_snapshot=path),
_workspace_id="sales",
)
context = load_schema_context(cfg)
assert context.physical.database == "warehouse"
assert context.physical.tables["orders"].comment == "Customer orders"
assert context.physical.tables["customers"].columns["id"].eligibility_reason == "sensitive"
assert context.effective_relationships["orders"][0].ref_table == "customers"
snapshot = load_catalog_metadata_snapshot(path, "sales")
records = catalog_schema_records(snapshot)
assert [record.kind for record in records].count("schema_relationship") == 1
relationship = next(record for record in records if record.kind == "schema_relationship")
assert relationship.metadata["tables"] == ["orders", "customers"]
assert qdrant_semantic_kind("schema_relationship") == "schema"
def test_relationship_hit_promotes_both_endpoint_tables():
result = SearchResult(
key="relationship:orders->customers",
label="orders->customers",
kind="schema_relationship",
signals={"vector": {"rank": 1, "score": 0.9}},
rrf=0.5,
)
assert schema_tables([result], 10) == [("customers", 0.5), ("orders", 0.5)]
+2
View File
@@ -19,6 +19,8 @@ def _leaf_paths(command, prefix: tuple[str, ...] = ()) -> set[str]:
return {" ".join(prefix)} if prefix else set()
paths: set[str] = set()
for name, child in children.items():
if getattr(child, "hidden", False):
continue
paths.update(_leaf_paths(child, prefix + (name,)))
return paths
+4 -1
View File
@@ -314,7 +314,10 @@ resources:
assert isinstance(cfg.vectors, QdrantConfig)
assert cfg.vectors.base_url == "http://qdrant:6333"
assert cfg.vectors.collection == "psd-clinical"
assert cfg.vectors.collections == {
"reference": "psd-clinical",
"memory": "psd-clinical-memory",
}
@pytest.mark.parametrize(
+3 -1
View File
@@ -487,7 +487,9 @@ def test_active_searcher_splits_default_and_mixed_kinds_before_global_limit(tmp_
assert all(call[0] != ["evidence"] for call in calls)
calls.clear()
searcher.search([1.0], top_n=1)
assert calls[0][0] == ["memory", "schema_column", "schema_table", "solved_question"]
assert calls[0][0] == [
"memory", "schema_column", "schema_relationship", "schema_table", "solved_question"
]
assert all(call[0] is not None for call in calls)
+129
View File
@@ -60,3 +60,132 @@ def test_endpoint_change_changes_binding(tmp_path):
raw["dwh"] = {"type": "thoth_rest", "database": {"database": "warehouse", "schema": "dw"}, "endpoint": {"base_url": "http://other.example.invalid", "api_key": "secret"}}
cfg2 = _write_cfg(tmp_path, raw)
assert config_dwh_binding(cfg1)["config_fingerprint"] != config_dwh_binding(cfg2)["config_fingerprint"]
def test_catalog_database_and_metadata_revision_are_explicit_lsh_binding_inputs(tmp_path):
import json
from tht.jobs.dwh_pipeline import config_dwh_binding
snapshot = tmp_path / "catalog-metadata.json"
snapshot.write_text(json.dumps({
"schemaVersion": 1,
"workspaceId": "psd",
"databaseId": "database-1",
"databaseName": "warehouse",
"schemaName": "dw",
"metadataContentRevision": 7,
"tables": [],
"relationships": [],
}), encoding="utf-8")
raw = _cfg(tmp_path)
raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
binding = config_dwh_binding(_write_cfg(tmp_path, raw))
assert binding["workspace_id"] == "psd"
assert binding["catalog_database_id"] == "database-1"
assert binding["metadata_content_revision"] == 7
def test_catalog_pipeline_materializes_schema_and_bound_lsh_generation(monkeypatch, tmp_path):
import json
from tht.cli.preprocess_cmd import run_catalog_dwh_from_config
snapshot = tmp_path / "catalog-metadata.json"
snapshot.write_text(json.dumps({
"schemaVersion": 1,
"workspaceId": "psd",
"databaseId": "database-1",
"databaseName": "warehouse",
"schemaName": "dw",
"metadataContentRevision": 7,
"tables": [{
"id": "table-orders",
"name": "orders",
"description": "Orders from the Catalog",
"descriptionSource": "curated",
"columns": [{
"id": "column-id",
"name": "id",
"ordinalPosition": 1,
"dataType": "bigint",
"isNullable": False,
"defaultExpression": None,
"primaryKeyPosition": 1,
"sensitive": False,
"description": "Order identifier",
"descriptionSource": "generated",
}],
}],
"relationships": [],
}), encoding="utf-8")
raw = _cfg(tmp_path)
raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
cfg = _write_cfg(tmp_path, raw)
def build_lsh(_cfg, *, physical_file, output_dir):
assert "Orders from the Catalog" in physical_file.read_text(encoding="utf-8")
for name, value in (
("dw_lsh.pkl", b"lsh"),
("dw_minhashes.pkl", b"minhashes"),
("dw_meta.json", b"{}"),
):
(output_dir / name).write_bytes(value)
return ({"orders.id": "hash"}, [], [], {})
monkeypatch.setattr("tht.cli.lsh_cmd.build_lsh_artifacts", build_lsh)
report, result = run_catalog_dwh_from_config(cfg)
assert report.status == "succeeded"
assert result[0] == {"orders.id": "hash"}
owner = json.loads((tmp_path / ".tht-dwh" / "OWNER.json").read_text(encoding="utf-8"))
assert owner["schema_version"] == 2
assert owner["binding"]["workspace_id"] == "psd"
assert owner["binding"]["catalog_database_id"] == "database-1"
assert owner["binding"]["metadata_content_revision"] == 7
def test_catalog_pipeline_replaces_prior_derived_binding(monkeypatch, tmp_path):
import json
from tht.cli.preprocess_cmd import run_catalog_dwh_from_config
snapshot = tmp_path / "catalog-metadata.json"
def publish_snapshot(revision: int) -> None:
snapshot.write_text(json.dumps({
"schemaVersion": 1,
"workspaceId": "psd",
"databaseId": "database-1",
"databaseName": "warehouse",
"schemaName": "dw",
"metadataContentRevision": revision,
"tables": [],
"relationships": [],
}), encoding="utf-8")
publish_snapshot(7)
raw = _cfg(tmp_path)
raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
cfg = _write_cfg(tmp_path, raw)
def build_lsh(_cfg, *, physical_file, output_dir):
assert physical_file.is_file()
for name, value in (
("dw_lsh.pkl", b"lsh"),
("dw_minhashes.pkl", b"minhashes"),
("dw_meta.json", b"{}"),
):
(output_dir / name).write_bytes(value)
return ({}, [], [], {})
monkeypatch.setattr("tht.cli.lsh_cmd.build_lsh_artifacts", build_lsh)
first, _ = run_catalog_dwh_from_config(cfg)
assert first.status == "succeeded"
publish_snapshot(8)
second, _ = run_catalog_dwh_from_config(cfg)
assert second.status == "succeeded"
owner = json.loads((tmp_path / ".tht-dwh" / "OWNER.json").read_text(encoding="utf-8"))
assert owner["binding"]["metadata_content_revision"] == 8
+33
View File
@@ -93,6 +93,39 @@ def test_v2_valid_materialized_corpus_is_validated_before_preprocessing(monkeypa
assert calls[:2] == [("validate", tmp_path), ("vector", True)]
def test_clear_removes_only_derived_paths_and_preserves_memory(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
monkeypatch.setenv("THT_PROFILE", "server")
config = _runtime_config(tmp_path)
for path in (
tmp_path / ".tht-dwh",
tmp_path / ".tht-jobs",
tmp_path / "corpus",
tmp_path / "indexes" / "lsh",
):
path.mkdir(parents=True, exist_ok=True)
(path / "derived").write_text("generated", encoding="utf-8")
physical = tmp_path / "artifacts" / "mschema" / "physical.yaml"
physical.parent.mkdir(parents=True)
physical.write_text("generated", encoding="utf-8")
memory = tmp_path / "memory" / "registry.jsonl"
memory.parent.mkdir()
memory.write_text("durable", encoding="utf-8")
class Store:
def clear_reference(self):
return True
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda *_args, **_kwargs: Store())
counts = command.clear_from_config(config)
assert counts == {"referenceCollections": 1, "derivedPaths": 5}
assert memory.read_text(encoding="utf-8") == "durable"
assert not (tmp_path / ".tht-dwh").exists()
assert not (tmp_path / "indexes" / "lsh").exists()
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
+38 -25
View File
@@ -32,6 +32,9 @@ class _FakeVectorStore:
def existing_hashes(self, collection, kinds):
return {}
def delete_kinds(self, collection, kinds):
return 0
def upsert(self, collection, records):
self.upserts.append((collection, records))
return len(records)
@@ -58,6 +61,7 @@ roots:
sessions: {tmp_path / 'sessions'}
artifacts: {tmp_path / 'artifacts'}
indexes: {tmp_path / 'indexes'}
catalog_metadata_snapshot: {tmp_path / 'catalog-metadata.json'}
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
@@ -68,22 +72,34 @@ embeddings:
return cfg
def _write_schema_artifacts(tmp_path: Path) -> None:
(tmp_path / "artifacts" / "mschema").mkdir(parents=True, exist_ok=True)
(tmp_path / "artifacts" / "mschema" / "physical.yaml").write_text(
"""
database: analytics
schema: mart
introspected_at: 2026-01-01T00:00:00+00:00
tables:
fact_patient:
comment: Patients
columns:
id:
type: bigint
"""
)
(tmp_path / "artifacts" / "mschema" / "annotations.yaml").write_text("tables: {}\n")
def _write_catalog_snapshot(tmp_path: Path) -> None:
(tmp_path / "catalog-metadata.json").write_text(json.dumps({
"schemaVersion": 1,
"workspaceId": "psd-clinical",
"databaseId": "db-1",
"databaseName": "analytics",
"schemaName": "mart",
"metadataContentRevision": 1,
"tables": [{
"id": "table-1",
"name": "fact_patient",
"description": "Patients",
"descriptionSource": "curated",
"columns": [{
"id": "column-1",
"name": "id",
"ordinalPosition": 1,
"dataType": "bigint",
"isNullable": False,
"defaultExpression": None,
"primaryKeyPosition": 1,
"sensitive": False,
"description": None,
"descriptionSource": None,
}],
}],
"relationships": [],
}))
def _memory_record() -> MemoryRecord:
@@ -104,7 +120,7 @@ def _memory_record() -> MemoryRecord:
def test_vector_index_schema_accepts_qdrant_only_runtime_config(tmp_path, monkeypatch):
cfg = _qdrant_runtime_config(tmp_path)
_write_schema_artifacts(tmp_path)
_write_catalog_snapshot(tmp_path)
store = _FakeVectorStore()
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: store)
@@ -118,7 +134,7 @@ def test_vector_index_schema_accepts_qdrant_only_runtime_config(tmp_path, monkey
def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, monkeypatch):
cfg = _qdrant_runtime_config(tmp_path)
_write_schema_artifacts(tmp_path)
_write_catalog_snapshot(tmp_path)
store = _FakeVectorStore()
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: store)
@@ -131,12 +147,8 @@ def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, mo
assert json.loads(response.stdout) == {
"artifactIdentities": [
{
"digest": _sha_file(tmp_path / "artifacts" / "mschema" / "annotations.yaml"),
"kind": "schema_annotations",
},
{
"digest": _sha_file(tmp_path / "artifacts" / "mschema" / "physical.yaml"),
"kind": "physical_schema",
"digest": _sha_file(tmp_path / "catalog-metadata.json"),
"kind": "catalog_metadata_snapshot",
},
],
"code": "ok",
@@ -146,6 +158,7 @@ def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, mo
"columns": 1,
"deleted": 0,
"records": 2,
"relationships": 0,
"tables": 1,
"unchanged": 0,
"updated": 0,
@@ -160,7 +173,7 @@ def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, mo
def test_vector_index_schema_json_failure_is_pristine(tmp_path, monkeypatch):
cfg = _qdrant_runtime_config(tmp_path)
_write_schema_artifacts(tmp_path)
_write_catalog_snapshot(tmp_path)
def boom(cfg, require_write):
raise VectorStoreError("semantic_index_incompatible")
+188
View File
@@ -188,6 +188,194 @@ def _store(
)
def test_explicit_collections_route_reference_and_memory_independently():
calls = []
def request(method, url, *, json=None, timeout=None):
calls.append((method, url, json))
if method == "GET" and url.endswith("/collections/demo"):
return FakeResponse(404, {"status": "error"})
if method == "GET":
return FakeResponse(200, {
"result": {
"config": {"params": {"vectors": {"size": 1024, "distance": "Cosine"}}},
"payload_schema": {
field: {"data_type": "keyword"}
for field in (
"content_hash", "document_id", "kind", "record_key", "record_kind",
"vector_generation", "workspace_id", "workspace_revision",
)
},
}
})
if method == "POST" and url.endswith("/points/query"):
return FakeResponse(200, {"result": {"points": []}})
if method == "PUT" and "/points" in url:
return FakeResponse(200, {"result": {"status": "acknowledged"}})
raise AssertionError((method, url, json))
store = QdrantVectorStore(
base_url="http://qdrant:6333",
collections={"reference": "demo-reference", "memory": "demo-memory"},
workspace_id="demo",
workspace_revision="a" * 40,
expected_dimension=1024,
request=request,
)
store.search(["schema_records"], [0.2] * 1024, limit=1)
store.search(["memory"], [0.2] * 1024, limit=1)
store.upsert("memory", [_write_record("memory:1", "memory")])
urls = [url for _method, url, _payload in calls]
assert any("/collections/demo-reference/points/query" in url for url in urls)
assert any("/collections/demo-memory/points/query" in url for url in urls)
assert any("/collections/demo-memory/points" in url for url in urls)
assert not any("/collections/demo-reference/points?" in url for url in urls)
def test_clear_drops_reference_without_deleting_memory_collection():
calls = []
def request(method, url, *, json=None, timeout=None):
calls.append((method, url))
if method == "GET" and url.endswith("/collections/demo"):
return FakeResponse(404, {"status": "error"})
if method == "GET" and url.endswith("/collections/demo-reference"):
return FakeResponse(200, {"result": {}})
if method == "DELETE" and url.endswith("/collections/demo-reference"):
return FakeResponse(200, {"status": "ok"})
raise AssertionError((method, url, json))
store = QdrantVectorStore(
base_url="http://qdrant:6333",
collections={"reference": "demo-reference", "memory": "demo-memory"},
workspace_id="demo",
expected_dimension=1024,
request=request,
)
assert store.clear_reference() is True
assert ("DELETE", "http://qdrant:6333/collections/demo-reference") in calls
assert not any(method == "DELETE" and url.endswith("/collections/demo-memory") for method, url in calls)
def test_clear_can_create_memory_collection_before_payload_indexes_become_visible():
calls = []
memory_created = False
def request(method, url, *, json=None, timeout=None):
nonlocal memory_created
calls.append((method, url, json))
if method == "GET" and url.endswith("/collections/demo"):
return FakeResponse(200, {"result": {}})
if method == "GET" and url.endswith("/collections/demo-memory"):
if not memory_created:
return FakeResponse(404, {"status": "error"})
return FakeResponse(200, {
"result": {
"config": {"params": {"vectors": {"size": 1024, "distance": "Cosine"}}},
# Qdrant exposes newly-created payload indexes asynchronously.
"payload_schema": {},
}
})
if method == "PUT" and url.endswith("/collections/demo-memory"):
memory_created = True
return FakeResponse(200, {"status": "ok"})
if method == "PUT" and url.endswith("/collections/demo-memory/index"):
return FakeResponse(200, {"status": "ok"})
if method == "POST" and url.endswith("/collections/demo/points/scroll"):
return FakeResponse(200, {
"result": {"points": [], "next_page_offset": None}
})
if method == "DELETE" and url.endswith("/collections/demo"):
return FakeResponse(200, {"status": "ok"})
if method == "GET" and url.endswith("/collections/demo-reference"):
return FakeResponse(404, {"status": "error"})
raise AssertionError((method, url, json))
store = QdrantVectorStore(
base_url="http://qdrant:6333",
collections={"reference": "demo-reference", "memory": "demo-memory"},
workspace_id="demo",
expected_dimension=1024,
collection_lifecycle="require_existing",
request=request,
)
assert store.clear_reference() is True
assert memory_created is True
assert any(method == "DELETE" and url.endswith("/collections/demo") for method, url, _ in calls)
def test_clear_migrates_legacy_memory_before_retiring_shared_collection():
calls = []
legacy_point = {
"id": "legacy-memory-id",
"vector": [0.25] * 1024,
"payload": {
"workspace_id": "demo",
"kind": "memory",
"record_kind": "memory",
"record_key": "memory:legacy",
"content_hash": "sha256:" + "a" * 64,
},
}
ready_memory = {
"result": {
"config": {"params": {"vectors": {"size": 1024, "distance": "Cosine"}}},
"payload_schema": {
field: {"data_type": "keyword"}
for field in (
"content_hash", "document_id", "kind", "record_key", "record_kind",
"vector_generation", "workspace_id", "workspace_revision",
)
},
}
}
def request(method, url, *, json=None, timeout=None):
calls.append((method, url, json))
if method == "GET" and url.endswith("/collections/demo"):
return FakeResponse(200, {"result": {}})
if method == "GET" and url.endswith("/collections/demo-memory"):
return FakeResponse(200, ready_memory)
if method == "POST" and url.endswith("/collections/demo/points/scroll"):
assert json["with_vector"] is True
return FakeResponse(200, {
"result": {"points": [legacy_point], "next_page_offset": None}
})
if method == "PUT" and "/collections/demo-memory/points?wait=true" in url:
return FakeResponse(200, {"status": "ok"})
if method == "DELETE" and url.endswith("/collections/demo"):
return FakeResponse(200, {"status": "ok"})
if method == "GET" and url.endswith("/collections/demo-reference"):
return FakeResponse(200, {"result": {}})
if method == "DELETE" and url.endswith("/collections/demo-reference"):
return FakeResponse(200, {"status": "ok"})
raise AssertionError((method, url, json))
store = QdrantVectorStore(
base_url="http://qdrant:6333",
collections={"reference": "demo-reference", "memory": "demo-memory"},
workspace_id="demo",
expected_dimension=1024,
request=request,
)
assert store.clear_reference() is True
migrated = next(
payload for method, url, payload in calls
if method == "PUT" and "/collections/demo-memory/points?wait=true" in url
)
assert migrated == {"points": [legacy_point]}
deleted = [url for method, url, _payload in calls if method == "DELETE"]
assert deleted == [
"http://qdrant:6333/collections/demo",
"http://qdrant:6333/collections/demo-reference",
]
assert not any(url.endswith("/collections/demo-memory") for url in deleted)
def _ready_collection_with_bm25(fake: FakeQdrantHttp) -> None:
fake.collection = {"vectors": {"size": 1024, "distance": "Cosine"}}
fake.payload_indexes = {
@@ -81,11 +81,11 @@ def test_introspect_corrupt_catalog_falls_through(tmp_path):
assert "OK (cache)" not in res.output
def test_render_without_catalog_guides_fallback(tmp_path):
def test_render_without_schema_source_fails_closed(tmp_path):
cfg = _write_config(tmp_path)
res = CliRunner().invoke(app, ["schema", "render", "-c", str(cfg)])
assert res.exit_code == 1
assert "Esegui prima" in res.output
assert "physical schema is missing" in res.output
def test_examples_skip_one_unreadable_column_and_continue(caplog):
+2 -2
View File
@@ -36,7 +36,7 @@ class _FakeSearcher:
metadata={"session_id": "2026-01-01-000000-x", "sql": "SELECT 1",
"tables": ["fact_ablazione"], "question": "quanti pazienti nel 2024?"},
)]
if kinds == ["schema_table", "schema_column"]:
if kinds == ["schema_table", "schema_column", "schema_relationship"]:
return [SimpleNamespace(
kind="schema_table", ref="fact_ablazione", id="t1",
title="Tabella fact_ablazione", similarity=0.88,
@@ -128,7 +128,7 @@ def test_pack_single_embed_and_sections(tmp_path, monkeypatch):
assert res.exit_code == 0, res.output
assert emb.calls == 2 # schema/memory share one; Evidence embeds its own deterministic text
assert [call["kinds"] for call in searcher.calls] == [
["schema_table", "schema_column"],
["schema_table", "schema_column", "schema_relationship"],
["solved_question"],
]
assert "fact_ablazione" in res.output and "Ablazioni" in res.output