feat: complete catalog-driven preprocessing

2026-09-06 17:49:35 +02:00
parent 8707ae1d46
commit cffa60772e
141 changed files with 5898 additions and 3015 deletions
@@ -60,3 +60,132 @@ def test_endpoint_change_changes_binding(tmp_path):
    raw["dwh"] = {"type": "thoth_rest", "database": {"database": "warehouse", "schema": "dw"}, "endpoint": {"base_url": "http://other.example.invalid", "api_key": "secret"}}
    cfg2 = _write_cfg(tmp_path, raw)
    assert config_dwh_binding(cfg1)["config_fingerprint"] != config_dwh_binding(cfg2)["config_fingerprint"]
+
+
+def test_catalog_database_and_metadata_revision_are_explicit_lsh_binding_inputs(tmp_path):
+    import json
+    from tht.jobs.dwh_pipeline import config_dwh_binding
+
+    snapshot = tmp_path / "catalog-metadata.json"
+    snapshot.write_text(json.dumps({
+        "schemaVersion": 1,
+        "workspaceId": "psd",
+        "databaseId": "database-1",
+        "databaseName": "warehouse",
+        "schemaName": "dw",
+        "metadataContentRevision": 7,
+        "tables": [],
+        "relationships": [],
+    }), encoding="utf-8")
+    raw = _cfg(tmp_path)
+    raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
+    binding = config_dwh_binding(_write_cfg(tmp_path, raw))
+
+    assert binding["workspace_id"] == "psd"
+    assert binding["catalog_database_id"] == "database-1"
+    assert binding["metadata_content_revision"] == 7
+
+
+def test_catalog_pipeline_materializes_schema_and_bound_lsh_generation(monkeypatch, tmp_path):
+    import json
+
+    from tht.cli.preprocess_cmd import run_catalog_dwh_from_config
+
+    snapshot = tmp_path / "catalog-metadata.json"
+    snapshot.write_text(json.dumps({
+        "schemaVersion": 1,
+        "workspaceId": "psd",
+        "databaseId": "database-1",
+        "databaseName": "warehouse",
+        "schemaName": "dw",
+        "metadataContentRevision": 7,
+        "tables": [{
+            "id": "table-orders",
+            "name": "orders",
+            "description": "Orders from the Catalog",
+            "descriptionSource": "curated",
+            "columns": [{
+                "id": "column-id",
+                "name": "id",
+                "ordinalPosition": 1,
+                "dataType": "bigint",
+                "isNullable": False,
+                "defaultExpression": None,
+                "primaryKeyPosition": 1,
+                "sensitive": False,
+                "description": "Order identifier",
+                "descriptionSource": "generated",
+            }],
+        }],
+        "relationships": [],
+    }), encoding="utf-8")
+    raw = _cfg(tmp_path)
+    raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
+    cfg = _write_cfg(tmp_path, raw)
+
+    def build_lsh(_cfg, *, physical_file, output_dir):
+        assert "Orders from the Catalog" in physical_file.read_text(encoding="utf-8")
+        for name, value in (
+            ("dw_lsh.pkl", b"lsh"),
+            ("dw_minhashes.pkl", b"minhashes"),
+            ("dw_meta.json", b"{}"),
+        ):
+            (output_dir / name).write_bytes(value)
+        return ({"orders.id": "hash"}, [], [], {})
+
+    monkeypatch.setattr("tht.cli.lsh_cmd.build_lsh_artifacts", build_lsh)
+    report, result = run_catalog_dwh_from_config(cfg)
+
+    assert report.status == "succeeded"
+    assert result[0] == {"orders.id": "hash"}
+    owner = json.loads((tmp_path / ".tht-dwh" / "OWNER.json").read_text(encoding="utf-8"))
+    assert owner["schema_version"] == 2
+    assert owner["binding"]["workspace_id"] == "psd"
+    assert owner["binding"]["catalog_database_id"] == "database-1"
+    assert owner["binding"]["metadata_content_revision"] == 7
+
+
+def test_catalog_pipeline_replaces_prior_derived_binding(monkeypatch, tmp_path):
+    import json
+
+    from tht.cli.preprocess_cmd import run_catalog_dwh_from_config
+
+    snapshot = tmp_path / "catalog-metadata.json"
+
+    def publish_snapshot(revision: int) -> None:
+        snapshot.write_text(json.dumps({
+            "schemaVersion": 1,
+            "workspaceId": "psd",
+            "databaseId": "database-1",
+            "databaseName": "warehouse",
+            "schemaName": "dw",
+            "metadataContentRevision": revision,
+            "tables": [],
+            "relationships": [],
+        }), encoding="utf-8")
+
+    publish_snapshot(7)
+    raw = _cfg(tmp_path)
+    raw["paths"]["catalog_metadata_snapshot"] = str(snapshot)
+    cfg = _write_cfg(tmp_path, raw)
+
+    def build_lsh(_cfg, *, physical_file, output_dir):
+        assert physical_file.is_file()
+        for name, value in (
+            ("dw_lsh.pkl", b"lsh"),
+            ("dw_minhashes.pkl", b"minhashes"),
+            ("dw_meta.json", b"{}"),
+        ):
+            (output_dir / name).write_bytes(value)
+        return ({}, [], [], {})
+
+    monkeypatch.setattr("tht.cli.lsh_cmd.build_lsh_artifacts", build_lsh)
+    first, _ = run_catalog_dwh_from_config(cfg)
+    assert first.status == "succeeded"
+
+    publish_snapshot(8)
+    second, _ = run_catalog_dwh_from_config(cfg)
+
+    assert second.status == "succeeded"
+    owner = json.loads((tmp_path / ".tht-dwh" / "OWNER.json").read_text(encoding="utf-8"))
+    assert owner["binding"]["metadata_content_revision"] == 8