feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s
Publish documentation / publish (push) Successful in 2m12s
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tht.mschema.catalog_snapshot import load_catalog_metadata_snapshot
|
||||
from tht.mschema.context import load_schema_context
|
||||
from tht.search import SearchResult, schema_tables
|
||||
from tht.vectorstore.records import catalog_schema_records, qdrant_semantic_kind
|
||||
|
||||
|
||||
def _snapshot():
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"workspaceId": "sales",
|
||||
"databaseId": "db-1",
|
||||
"databaseName": "warehouse",
|
||||
"schemaName": "analytics",
|
||||
"metadataContentRevision": 7,
|
||||
"tables": [
|
||||
{
|
||||
"id": "table-orders",
|
||||
"name": "orders",
|
||||
"description": "Customer orders",
|
||||
"descriptionSource": "curated",
|
||||
"columns": [
|
||||
{
|
||||
"id": "column-customer",
|
||||
"name": "customer_id",
|
||||
"ordinalPosition": 1,
|
||||
"dataType": "uuid",
|
||||
"isNullable": False,
|
||||
"defaultExpression": None,
|
||||
"primaryKeyPosition": None,
|
||||
"sensitive": False,
|
||||
"description": "Ordering customer",
|
||||
"descriptionSource": "generated",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "table-customers",
|
||||
"name": "customers",
|
||||
"description": None,
|
||||
"descriptionSource": None,
|
||||
"columns": [
|
||||
{
|
||||
"id": "column-id",
|
||||
"name": "id",
|
||||
"ordinalPosition": 1,
|
||||
"dataType": "uuid",
|
||||
"isNullable": False,
|
||||
"defaultExpression": None,
|
||||
"primaryKeyPosition": 1,
|
||||
"sensitive": True,
|
||||
"description": "Customer identifier",
|
||||
"descriptionSource": "source_comment",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"id": "relationship-1",
|
||||
"origin": "physical",
|
||||
"sourceTable": "orders",
|
||||
"sourceColumns": ["customer_id"],
|
||||
"targetTable": "customers",
|
||||
"targetColumns": ["id"],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_catalog_snapshot_is_the_schema_context_and_vector_source(tmp_path):
|
||||
path = tmp_path / "catalog-metadata.json"
|
||||
path.write_text(json.dumps(_snapshot()))
|
||||
cfg = SimpleNamespace(
|
||||
paths=SimpleNamespace(catalog_metadata_snapshot=path),
|
||||
_workspace_id="sales",
|
||||
)
|
||||
|
||||
context = load_schema_context(cfg)
|
||||
assert context.physical.database == "warehouse"
|
||||
assert context.physical.tables["orders"].comment == "Customer orders"
|
||||
assert context.physical.tables["customers"].columns["id"].eligibility_reason == "sensitive"
|
||||
assert context.effective_relationships["orders"][0].ref_table == "customers"
|
||||
|
||||
snapshot = load_catalog_metadata_snapshot(path, "sales")
|
||||
records = catalog_schema_records(snapshot)
|
||||
assert [record.kind for record in records].count("schema_relationship") == 1
|
||||
relationship = next(record for record in records if record.kind == "schema_relationship")
|
||||
assert relationship.metadata["tables"] == ["orders", "customers"]
|
||||
assert qdrant_semantic_kind("schema_relationship") == "schema"
|
||||
|
||||
|
||||
def test_relationship_hit_promotes_both_endpoint_tables():
|
||||
result = SearchResult(
|
||||
key="relationship:orders->customers",
|
||||
label="orders->customers",
|
||||
kind="schema_relationship",
|
||||
signals={"vector": {"rank": 1, "score": 0.9}},
|
||||
rrf=0.5,
|
||||
)
|
||||
assert schema_tables([result], 10) == [("customers", 0.5), ("orders", 0.5)]
|
||||
@@ -19,6 +19,8 @@ def _leaf_paths(command, prefix: tuple[str, ...] = ()) -> set[str]:
|
||||
return {" ".join(prefix)} if prefix else set()
|
||||
paths: set[str] = set()
|
||||
for name, child in children.items():
|
||||
if getattr(child, "hidden", False):
|
||||
continue
|
||||
paths.update(_leaf_paths(child, prefix + (name,)))
|
||||
return paths
|
||||
|
||||
|
||||
@@ -314,7 +314,10 @@ resources:
|
||||
|
||||
assert isinstance(cfg.vectors, QdrantConfig)
|
||||
assert cfg.vectors.base_url == "http://qdrant:6333"
|
||||
assert cfg.vectors.collection == "psd-clinical"
|
||||
assert cfg.vectors.collections == {
|
||||
"reference": "psd-clinical",
|
||||
"memory": "psd-clinical-memory",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -487,7 +487,9 @@ def test_active_searcher_splits_default_and_mixed_kinds_before_global_limit(tmp_
|
||||
assert all(call[0] != ["evidence"] for call in calls)
|
||||
calls.clear()
|
||||
searcher.search([1.0], top_n=1)
|
||||
assert calls[0][0] == ["memory", "schema_column", "schema_table", "solved_question"]
|
||||
assert calls[0][0] == [
|
||||
"memory", "schema_column", "schema_relationship", "schema_table", "solved_question"
|
||||
]
|
||||
assert all(call[0] is not None for call in calls)
|
||||
|
||||
|
||||
|
||||
@@ -60,3 +60,132 @@ def test_endpoint_change_changes_binding(tmp_path):
|
||||
raw["dwh"] = {"type": "thoth_rest", "database": {"database": "warehouse", "schema": "dw"}, "endpoint": {"base_url": "http://other.example.invalid", "api_key": "secret"}}
|
||||
cfg2 = _write_cfg(tmp_path, raw)
|
||||
assert config_dwh_binding(cfg1)["config_fingerprint"] != config_dwh_binding(cfg2)["config_fingerprint"]
|
||||
|
||||
|
||||
def test_catalog_database_and_metadata_revision_are_explicit_lsh_binding_inputs(tmp_path):
|
||||
import json
|
||||
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||
|
||||
snapshot = tmp_path / "catalog-metadata.json"
|
||||
snapshot.write_text(json.dumps({
|
||||
"schemaVersion": 1,
|
||||
"workspaceId": "psd",
|
||||
"databaseId": "database-1",
|
||||
"databaseName": "warehouse",
|
||||
"schemaName": "dw",
|
||||
"metadataContentRevision": 7,
|
||||
"tables": [],
|
||||
"relationships": [],
|
||||
}), encoding="utf-8")
|
||||
raw = _cfg(tmp_path)
|
||||
raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
|
||||
binding = config_dwh_binding(_write_cfg(tmp_path, raw))
|
||||
|
||||
assert binding["workspace_id"] == "psd"
|
||||
assert binding["catalog_database_id"] == "database-1"
|
||||
assert binding["metadata_content_revision"] == 7
|
||||
|
||||
|
||||
def test_catalog_pipeline_materializes_schema_and_bound_lsh_generation(monkeypatch, tmp_path):
|
||||
import json
|
||||
|
||||
from tht.cli.preprocess_cmd import run_catalog_dwh_from_config
|
||||
|
||||
snapshot = tmp_path / "catalog-metadata.json"
|
||||
snapshot.write_text(json.dumps({
|
||||
"schemaVersion": 1,
|
||||
"workspaceId": "psd",
|
||||
"databaseId": "database-1",
|
||||
"databaseName": "warehouse",
|
||||
"schemaName": "dw",
|
||||
"metadataContentRevision": 7,
|
||||
"tables": [{
|
||||
"id": "table-orders",
|
||||
"name": "orders",
|
||||
"description": "Orders from the Catalog",
|
||||
"descriptionSource": "curated",
|
||||
"columns": [{
|
||||
"id": "column-id",
|
||||
"name": "id",
|
||||
"ordinalPosition": 1,
|
||||
"dataType": "bigint",
|
||||
"isNullable": False,
|
||||
"defaultExpression": None,
|
||||
"primaryKeyPosition": 1,
|
||||
"sensitive": False,
|
||||
"description": "Order identifier",
|
||||
"descriptionSource": "generated",
|
||||
}],
|
||||
}],
|
||||
"relationships": [],
|
||||
}), encoding="utf-8")
|
||||
raw = _cfg(tmp_path)
|
||||
raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
|
||||
cfg = _write_cfg(tmp_path, raw)
|
||||
|
||||
def build_lsh(_cfg, *, physical_file, output_dir):
|
||||
assert "Orders from the Catalog" in physical_file.read_text(encoding="utf-8")
|
||||
for name, value in (
|
||||
("dw_lsh.pkl", b"lsh"),
|
||||
("dw_minhashes.pkl", b"minhashes"),
|
||||
("dw_meta.json", b"{}"),
|
||||
):
|
||||
(output_dir / name).write_bytes(value)
|
||||
return ({"orders.id": "hash"}, [], [], {})
|
||||
|
||||
monkeypatch.setattr("tht.cli.lsh_cmd.build_lsh_artifacts", build_lsh)
|
||||
report, result = run_catalog_dwh_from_config(cfg)
|
||||
|
||||
assert report.status == "succeeded"
|
||||
assert result[0] == {"orders.id": "hash"}
|
||||
owner = json.loads((tmp_path / ".tht-dwh" / "OWNER.json").read_text(encoding="utf-8"))
|
||||
assert owner["schema_version"] == 2
|
||||
assert owner["binding"]["workspace_id"] == "psd"
|
||||
assert owner["binding"]["catalog_database_id"] == "database-1"
|
||||
assert owner["binding"]["metadata_content_revision"] == 7
|
||||
|
||||
|
||||
def test_catalog_pipeline_replaces_prior_derived_binding(monkeypatch, tmp_path):
|
||||
import json
|
||||
|
||||
from tht.cli.preprocess_cmd import run_catalog_dwh_from_config
|
||||
|
||||
snapshot = tmp_path / "catalog-metadata.json"
|
||||
|
||||
def publish_snapshot(revision: int) -> None:
|
||||
snapshot.write_text(json.dumps({
|
||||
"schemaVersion": 1,
|
||||
"workspaceId": "psd",
|
||||
"databaseId": "database-1",
|
||||
"databaseName": "warehouse",
|
||||
"schemaName": "dw",
|
||||
"metadataContentRevision": revision,
|
||||
"tables": [],
|
||||
"relationships": [],
|
||||
}), encoding="utf-8")
|
||||
|
||||
publish_snapshot(7)
|
||||
raw = _cfg(tmp_path)
|
||||
raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
|
||||
cfg = _write_cfg(tmp_path, raw)
|
||||
|
||||
def build_lsh(_cfg, *, physical_file, output_dir):
|
||||
assert physical_file.is_file()
|
||||
for name, value in (
|
||||
("dw_lsh.pkl", b"lsh"),
|
||||
("dw_minhashes.pkl", b"minhashes"),
|
||||
("dw_meta.json", b"{}"),
|
||||
):
|
||||
(output_dir / name).write_bytes(value)
|
||||
return ({}, [], [], {})
|
||||
|
||||
monkeypatch.setattr("tht.cli.lsh_cmd.build_lsh_artifacts", build_lsh)
|
||||
first, _ = run_catalog_dwh_from_config(cfg)
|
||||
assert first.status == "succeeded"
|
||||
|
||||
publish_snapshot(8)
|
||||
second, _ = run_catalog_dwh_from_config(cfg)
|
||||
|
||||
assert second.status == "succeeded"
|
||||
owner = json.loads((tmp_path / ".tht-dwh" / "OWNER.json").read_text(encoding="utf-8"))
|
||||
assert owner["binding"]["metadata_content_revision"] == 8
|
||||
|
||||
@@ -93,6 +93,39 @@ def test_v2_valid_materialized_corpus_is_validated_before_preprocessing(monkeypa
|
||||
assert calls[:2] == [("validate", tmp_path), ("vector", True)]
|
||||
|
||||
|
||||
def test_clear_removes_only_derived_paths_and_preserves_memory(monkeypatch, tmp_path):
|
||||
import tht.cli.preprocess_cmd as command
|
||||
|
||||
monkeypatch.setenv("THT_PROFILE", "server")
|
||||
config = _runtime_config(tmp_path)
|
||||
for path in (
|
||||
tmp_path / ".tht-dwh",
|
||||
tmp_path / ".tht-jobs",
|
||||
tmp_path / "corpus",
|
||||
tmp_path / "indexes" / "lsh",
|
||||
):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
(path / "derived").write_text("generated", encoding="utf-8")
|
||||
physical = tmp_path / "artifacts" / "mschema" / "physical.yaml"
|
||||
physical.parent.mkdir(parents=True)
|
||||
physical.write_text("generated", encoding="utf-8")
|
||||
memory = tmp_path / "memory" / "registry.jsonl"
|
||||
memory.parent.mkdir()
|
||||
memory.write_text("durable", encoding="utf-8")
|
||||
|
||||
class Store:
|
||||
def clear_reference(self):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda *_args, **_kwargs: Store())
|
||||
counts = command.clear_from_config(config)
|
||||
|
||||
assert counts == {"referenceCollections": 1, "derivedPaths": 5}
|
||||
assert memory.read_text(encoding="utf-8") == "durable"
|
||||
assert not (tmp_path / ".tht-dwh").exists()
|
||||
assert not (tmp_path / "indexes" / "lsh").exists()
|
||||
|
||||
|
||||
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
|
||||
import tht.cli.preprocess_cmd as command
|
||||
|
||||
|
||||
@@ -32,6 +32,9 @@ class _FakeVectorStore:
|
||||
def existing_hashes(self, collection, kinds):
|
||||
return {}
|
||||
|
||||
def delete_kinds(self, collection, kinds):
|
||||
return 0
|
||||
|
||||
def upsert(self, collection, records):
|
||||
self.upserts.append((collection, records))
|
||||
return len(records)
|
||||
@@ -58,6 +61,7 @@ roots:
|
||||
sessions: {tmp_path / 'sessions'}
|
||||
artifacts: {tmp_path / 'artifacts'}
|
||||
indexes: {tmp_path / 'indexes'}
|
||||
catalog_metadata_snapshot: {tmp_path / 'catalog-metadata.json'}
|
||||
embeddings:
|
||||
provider: ollama_internal
|
||||
base_url: http://embedding:11434
|
||||
@@ -68,22 +72,34 @@ embeddings:
|
||||
return cfg
|
||||
|
||||
|
||||
def _write_schema_artifacts(tmp_path: Path) -> None:
|
||||
(tmp_path / "artifacts" / "mschema").mkdir(parents=True, exist_ok=True)
|
||||
(tmp_path / "artifacts" / "mschema" / "physical.yaml").write_text(
|
||||
"""
|
||||
database: analytics
|
||||
schema: mart
|
||||
introspected_at: 2026-01-01T00:00:00+00:00
|
||||
tables:
|
||||
fact_patient:
|
||||
comment: Patients
|
||||
columns:
|
||||
id:
|
||||
type: bigint
|
||||
"""
|
||||
)
|
||||
(tmp_path / "artifacts" / "mschema" / "annotations.yaml").write_text("tables: {}\n")
|
||||
def _write_catalog_snapshot(tmp_path: Path) -> None:
|
||||
(tmp_path / "catalog-metadata.json").write_text(json.dumps({
|
||||
"schemaVersion": 1,
|
||||
"workspaceId": "psd-clinical",
|
||||
"databaseId": "db-1",
|
||||
"databaseName": "analytics",
|
||||
"schemaName": "mart",
|
||||
"metadataContentRevision": 1,
|
||||
"tables": [{
|
||||
"id": "table-1",
|
||||
"name": "fact_patient",
|
||||
"description": "Patients",
|
||||
"descriptionSource": "curated",
|
||||
"columns": [{
|
||||
"id": "column-1",
|
||||
"name": "id",
|
||||
"ordinalPosition": 1,
|
||||
"dataType": "bigint",
|
||||
"isNullable": False,
|
||||
"defaultExpression": None,
|
||||
"primaryKeyPosition": 1,
|
||||
"sensitive": False,
|
||||
"description": None,
|
||||
"descriptionSource": None,
|
||||
}],
|
||||
}],
|
||||
"relationships": [],
|
||||
}))
|
||||
|
||||
|
||||
def _memory_record() -> MemoryRecord:
|
||||
@@ -104,7 +120,7 @@ def _memory_record() -> MemoryRecord:
|
||||
|
||||
def test_vector_index_schema_accepts_qdrant_only_runtime_config(tmp_path, monkeypatch):
|
||||
cfg = _qdrant_runtime_config(tmp_path)
|
||||
_write_schema_artifacts(tmp_path)
|
||||
_write_catalog_snapshot(tmp_path)
|
||||
store = _FakeVectorStore()
|
||||
|
||||
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: store)
|
||||
@@ -118,7 +134,7 @@ def test_vector_index_schema_accepts_qdrant_only_runtime_config(tmp_path, monkey
|
||||
|
||||
def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, monkeypatch):
|
||||
cfg = _qdrant_runtime_config(tmp_path)
|
||||
_write_schema_artifacts(tmp_path)
|
||||
_write_catalog_snapshot(tmp_path)
|
||||
store = _FakeVectorStore()
|
||||
|
||||
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: store)
|
||||
@@ -131,12 +147,8 @@ def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, mo
|
||||
assert json.loads(response.stdout) == {
|
||||
"artifactIdentities": [
|
||||
{
|
||||
"digest": _sha_file(tmp_path / "artifacts" / "mschema" / "annotations.yaml"),
|
||||
"kind": "schema_annotations",
|
||||
},
|
||||
{
|
||||
"digest": _sha_file(tmp_path / "artifacts" / "mschema" / "physical.yaml"),
|
||||
"kind": "physical_schema",
|
||||
"digest": _sha_file(tmp_path / "catalog-metadata.json"),
|
||||
"kind": "catalog_metadata_snapshot",
|
||||
},
|
||||
],
|
||||
"code": "ok",
|
||||
@@ -146,6 +158,7 @@ def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, mo
|
||||
"columns": 1,
|
||||
"deleted": 0,
|
||||
"records": 2,
|
||||
"relationships": 0,
|
||||
"tables": 1,
|
||||
"unchanged": 0,
|
||||
"updated": 0,
|
||||
@@ -160,7 +173,7 @@ def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, mo
|
||||
|
||||
def test_vector_index_schema_json_failure_is_pristine(tmp_path, monkeypatch):
|
||||
cfg = _qdrant_runtime_config(tmp_path)
|
||||
_write_schema_artifacts(tmp_path)
|
||||
_write_catalog_snapshot(tmp_path)
|
||||
|
||||
def boom(cfg, require_write):
|
||||
raise VectorStoreError("semantic_index_incompatible")
|
||||
|
||||
@@ -188,6 +188,194 @@ def _store(
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_collections_route_reference_and_memory_independently():
|
||||
calls = []
|
||||
|
||||
def request(method, url, *, json=None, timeout=None):
|
||||
calls.append((method, url, json))
|
||||
if method == "GET" and url.endswith("/collections/demo"):
|
||||
return FakeResponse(404, {"status": "error"})
|
||||
if method == "GET":
|
||||
return FakeResponse(200, {
|
||||
"result": {
|
||||
"config": {"params": {"vectors": {"size": 1024, "distance": "Cosine"}}},
|
||||
"payload_schema": {
|
||||
field: {"data_type": "keyword"}
|
||||
for field in (
|
||||
"content_hash", "document_id", "kind", "record_key", "record_kind",
|
||||
"vector_generation", "workspace_id", "workspace_revision",
|
||||
)
|
||||
},
|
||||
}
|
||||
})
|
||||
if method == "POST" and url.endswith("/points/query"):
|
||||
return FakeResponse(200, {"result": {"points": []}})
|
||||
if method == "PUT" and "/points" in url:
|
||||
return FakeResponse(200, {"result": {"status": "acknowledged"}})
|
||||
raise AssertionError((method, url, json))
|
||||
|
||||
store = QdrantVectorStore(
|
||||
base_url="http://qdrant:6333",
|
||||
collections={"reference": "demo-reference", "memory": "demo-memory"},
|
||||
workspace_id="demo",
|
||||
workspace_revision="a" * 40,
|
||||
expected_dimension=1024,
|
||||
request=request,
|
||||
)
|
||||
store.search(["schema_records"], [0.2] * 1024, limit=1)
|
||||
store.search(["memory"], [0.2] * 1024, limit=1)
|
||||
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||
|
||||
urls = [url for _method, url, _payload in calls]
|
||||
assert any("/collections/demo-reference/points/query" in url for url in urls)
|
||||
assert any("/collections/demo-memory/points/query" in url for url in urls)
|
||||
assert any("/collections/demo-memory/points" in url for url in urls)
|
||||
assert not any("/collections/demo-reference/points?" in url for url in urls)
|
||||
|
||||
|
||||
def test_clear_drops_reference_without_deleting_memory_collection():
|
||||
calls = []
|
||||
|
||||
def request(method, url, *, json=None, timeout=None):
|
||||
calls.append((method, url))
|
||||
if method == "GET" and url.endswith("/collections/demo"):
|
||||
return FakeResponse(404, {"status": "error"})
|
||||
if method == "GET" and url.endswith("/collections/demo-reference"):
|
||||
return FakeResponse(200, {"result": {}})
|
||||
if method == "DELETE" and url.endswith("/collections/demo-reference"):
|
||||
return FakeResponse(200, {"status": "ok"})
|
||||
raise AssertionError((method, url, json))
|
||||
|
||||
store = QdrantVectorStore(
|
||||
base_url="http://qdrant:6333",
|
||||
collections={"reference": "demo-reference", "memory": "demo-memory"},
|
||||
workspace_id="demo",
|
||||
expected_dimension=1024,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert store.clear_reference() is True
|
||||
assert ("DELETE", "http://qdrant:6333/collections/demo-reference") in calls
|
||||
assert not any(method == "DELETE" and url.endswith("/collections/demo-memory") for method, url in calls)
|
||||
|
||||
|
||||
def test_clear_can_create_memory_collection_before_payload_indexes_become_visible():
|
||||
calls = []
|
||||
memory_created = False
|
||||
|
||||
def request(method, url, *, json=None, timeout=None):
|
||||
nonlocal memory_created
|
||||
calls.append((method, url, json))
|
||||
if method == "GET" and url.endswith("/collections/demo"):
|
||||
return FakeResponse(200, {"result": {}})
|
||||
if method == "GET" and url.endswith("/collections/demo-memory"):
|
||||
if not memory_created:
|
||||
return FakeResponse(404, {"status": "error"})
|
||||
return FakeResponse(200, {
|
||||
"result": {
|
||||
"config": {"params": {"vectors": {"size": 1024, "distance": "Cosine"}}},
|
||||
# Qdrant exposes newly-created payload indexes asynchronously.
|
||||
"payload_schema": {},
|
||||
}
|
||||
})
|
||||
if method == "PUT" and url.endswith("/collections/demo-memory"):
|
||||
memory_created = True
|
||||
return FakeResponse(200, {"status": "ok"})
|
||||
if method == "PUT" and url.endswith("/collections/demo-memory/index"):
|
||||
return FakeResponse(200, {"status": "ok"})
|
||||
if method == "POST" and url.endswith("/collections/demo/points/scroll"):
|
||||
return FakeResponse(200, {
|
||||
"result": {"points": [], "next_page_offset": None}
|
||||
})
|
||||
if method == "DELETE" and url.endswith("/collections/demo"):
|
||||
return FakeResponse(200, {"status": "ok"})
|
||||
if method == "GET" and url.endswith("/collections/demo-reference"):
|
||||
return FakeResponse(404, {"status": "error"})
|
||||
raise AssertionError((method, url, json))
|
||||
|
||||
store = QdrantVectorStore(
|
||||
base_url="http://qdrant:6333",
|
||||
collections={"reference": "demo-reference", "memory": "demo-memory"},
|
||||
workspace_id="demo",
|
||||
expected_dimension=1024,
|
||||
collection_lifecycle="require_existing",
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert store.clear_reference() is True
|
||||
assert memory_created is True
|
||||
assert any(method == "DELETE" and url.endswith("/collections/demo") for method, url, _ in calls)
|
||||
|
||||
|
||||
def test_clear_migrates_legacy_memory_before_retiring_shared_collection():
|
||||
calls = []
|
||||
legacy_point = {
|
||||
"id": "legacy-memory-id",
|
||||
"vector": [0.25] * 1024,
|
||||
"payload": {
|
||||
"workspace_id": "demo",
|
||||
"kind": "memory",
|
||||
"record_kind": "memory",
|
||||
"record_key": "memory:legacy",
|
||||
"content_hash": "sha256:" + "a" * 64,
|
||||
},
|
||||
}
|
||||
ready_memory = {
|
||||
"result": {
|
||||
"config": {"params": {"vectors": {"size": 1024, "distance": "Cosine"}}},
|
||||
"payload_schema": {
|
||||
field: {"data_type": "keyword"}
|
||||
for field in (
|
||||
"content_hash", "document_id", "kind", "record_key", "record_kind",
|
||||
"vector_generation", "workspace_id", "workspace_revision",
|
||||
)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
def request(method, url, *, json=None, timeout=None):
|
||||
calls.append((method, url, json))
|
||||
if method == "GET" and url.endswith("/collections/demo"):
|
||||
return FakeResponse(200, {"result": {}})
|
||||
if method == "GET" and url.endswith("/collections/demo-memory"):
|
||||
return FakeResponse(200, ready_memory)
|
||||
if method == "POST" and url.endswith("/collections/demo/points/scroll"):
|
||||
assert json["with_vector"] is True
|
||||
return FakeResponse(200, {
|
||||
"result": {"points": [legacy_point], "next_page_offset": None}
|
||||
})
|
||||
if method == "PUT" and "/collections/demo-memory/points?wait=true" in url:
|
||||
return FakeResponse(200, {"status": "ok"})
|
||||
if method == "DELETE" and url.endswith("/collections/demo"):
|
||||
return FakeResponse(200, {"status": "ok"})
|
||||
if method == "GET" and url.endswith("/collections/demo-reference"):
|
||||
return FakeResponse(200, {"result": {}})
|
||||
if method == "DELETE" and url.endswith("/collections/demo-reference"):
|
||||
return FakeResponse(200, {"status": "ok"})
|
||||
raise AssertionError((method, url, json))
|
||||
|
||||
store = QdrantVectorStore(
|
||||
base_url="http://qdrant:6333",
|
||||
collections={"reference": "demo-reference", "memory": "demo-memory"},
|
||||
workspace_id="demo",
|
||||
expected_dimension=1024,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert store.clear_reference() is True
|
||||
migrated = next(
|
||||
payload for method, url, payload in calls
|
||||
if method == "PUT" and "/collections/demo-memory/points?wait=true" in url
|
||||
)
|
||||
assert migrated == {"points": [legacy_point]}
|
||||
deleted = [url for method, url, _payload in calls if method == "DELETE"]
|
||||
assert deleted == [
|
||||
"http://qdrant:6333/collections/demo",
|
||||
"http://qdrant:6333/collections/demo-reference",
|
||||
]
|
||||
assert not any(url.endswith("/collections/demo-memory") for url in deleted)
|
||||
|
||||
|
||||
def _ready_collection_with_bm25(fake: FakeQdrantHttp) -> None:
|
||||
fake.collection = {"vectors": {"size": 1024, "distance": "Cosine"}}
|
||||
fake.payload_indexes = {
|
||||
|
||||
@@ -81,11 +81,11 @@ def test_introspect_corrupt_catalog_falls_through(tmp_path):
|
||||
assert "OK (cache)" not in res.output
|
||||
|
||||
|
||||
def test_render_without_catalog_guides_fallback(tmp_path):
|
||||
def test_render_without_schema_source_fails_closed(tmp_path):
|
||||
cfg = _write_config(tmp_path)
|
||||
res = CliRunner().invoke(app, ["schema", "render", "-c", str(cfg)])
|
||||
assert res.exit_code == 1
|
||||
assert "Esegui prima" in res.output
|
||||
assert "physical schema is missing" in res.output
|
||||
|
||||
|
||||
def test_examples_skip_one_unreadable_column_and_continue(caplog):
|
||||
|
||||
@@ -36,7 +36,7 @@ class _FakeSearcher:
|
||||
metadata={"session_id": "2026-01-01-000000-x", "sql": "SELECT 1",
|
||||
"tables": ["fact_ablazione"], "question": "quanti pazienti nel 2024?"},
|
||||
)]
|
||||
if kinds == ["schema_table", "schema_column"]:
|
||||
if kinds == ["schema_table", "schema_column", "schema_relationship"]:
|
||||
return [SimpleNamespace(
|
||||
kind="schema_table", ref="fact_ablazione", id="t1",
|
||||
title="Tabella fact_ablazione", similarity=0.88,
|
||||
@@ -128,7 +128,7 @@ def test_pack_single_embed_and_sections(tmp_path, monkeypatch):
|
||||
assert res.exit_code == 0, res.output
|
||||
assert emb.calls == 2 # schema/memory share one; Evidence embeds its own deterministic text
|
||||
assert [call["kinds"] for call in searcher.calls] == [
|
||||
["schema_table", "schema_column"],
|
||||
["schema_table", "schema_column", "schema_relationship"],
|
||||
["solved_question"],
|
||||
]
|
||||
assert "fact_ablazione" in res.output and "Ablazioni" in res.output
|
||||
|
||||
Reference in New Issue
Block a user