feat: pristine harness JSON interfaces and require-existing semantic mode (P2)
This commit is contained in:
@@ -8,7 +8,11 @@ def test_local_compose_uses_the_generic_external_endpoint_contract():
|
|||||||
compose = yaml.safe_load((root / "compose.yaml").read_text())
|
compose = yaml.safe_load((root / "compose.yaml").read_text())
|
||||||
local = yaml.safe_load((root / "deploy/compose.local.yaml").read_text())
|
local = yaml.safe_load((root / "deploy/compose.local.yaml").read_text())
|
||||||
|
|
||||||
assert set(compose["services"]) == {"core", "frontend", "qdrant", "embedding", "embedding-model-init"}
|
assert set(compose["services"]) == {
|
||||||
|
"core", "frontend", "qdrant", "embedding", "embedding-model-init", "workspace-maintenance",
|
||||||
|
}
|
||||||
|
# workspace-maintenance is profile-gated: it must not be part of the default local startup.
|
||||||
|
assert compose["services"]["workspace-maintenance"].get("profiles") == ["workspace-maintenance"]
|
||||||
assert local["services"]["core"]["environment"]["AUTH_MODE"] == "none"
|
assert local["services"]["core"]["environment"]["AUTH_MODE"] == "none"
|
||||||
assert local["services"]["core"]["ports"] == ["127.0.0.1:${THOTH_CORE_HTTP_PORT:-8787}:8787"]
|
assert local["services"]["core"]["ports"] == ["127.0.0.1:${THOTH_CORE_HTTP_PORT:-8787}:8787"]
|
||||||
assert local["services"]["frontend"]["ports"] == ["127.0.0.1:${THOTH_HTTP_PORT:-8080}:8080"]
|
assert local["services"]["frontend"]["ports"] == ["127.0.0.1:${THOTH_HTTP_PORT:-8080}:8080"]
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import json
|
import json
|
||||||
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from typer.testing import CliRunner
|
from typer.testing import CliRunner
|
||||||
@@ -6,68 +7,152 @@ from typer.testing import CliRunner
|
|||||||
from tht.cli import app
|
from tht.cli import app
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_config(tmp_path: Path, name: str = "workspace.yaml") -> Path:
|
||||||
|
path = tmp_path / name
|
||||||
|
(tmp_path / "evidence").mkdir(exist_ok=True)
|
||||||
|
path.write_text(
|
||||||
|
f"""
|
||||||
|
runtime_identity:
|
||||||
|
workspace_id: psd-clinical
|
||||||
|
workspace_revision: {'a' * 40}
|
||||||
|
dwh:
|
||||||
|
type: postgres_direct
|
||||||
|
connection: {{database: analytics, schema: mart, user: reader, password: secret}}
|
||||||
|
vectors:
|
||||||
|
type: qdrant
|
||||||
|
base_url: http://qdrant:6333
|
||||||
|
collection: psd-clinical
|
||||||
|
embeddings:
|
||||||
|
provider: ollama_internal
|
||||||
|
base_url: http://embedding:11434
|
||||||
|
model: qwen3-embedding:0.6b
|
||||||
|
dim: 1024
|
||||||
|
evidence:
|
||||||
|
sources:
|
||||||
|
- type: filesystem
|
||||||
|
root: {tmp_path / 'evidence'}
|
||||||
|
roots:
|
||||||
|
sessions: {tmp_path / 'sessions'}
|
||||||
|
artifacts: {tmp_path / 'artifacts'}
|
||||||
|
indexes: {tmp_path / 'indexes'}
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
|
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
|
||||||
import tht.cli.preprocess_cmd as command
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
result = SimpleNamespace(model_dump=lambda mode=None: {
|
config = _runtime_config(tmp_path)
|
||||||
"status": "succeeded", "generation": "gen:abc", "published": True
|
result = SimpleNamespace(
|
||||||
})
|
model_dump=lambda mode=None: {
|
||||||
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
"status": "succeeded",
|
||||||
response = CliRunner().invoke(
|
"generation": "gen:abc",
|
||||||
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
|
"published": True,
|
||||||
|
"counts": {"changed": 0, "unchanged": 0, "removed": 0, "documents": 0, "chunks": 0},
|
||||||
|
"changed": [],
|
||||||
|
"unchanged": [],
|
||||||
|
"removed": [],
|
||||||
|
"manifest_id": "manifest-1",
|
||||||
|
"run_id": "a" * 32,
|
||||||
|
"resumed_from": None,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
||||||
|
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
|
||||||
assert response.exit_code == 0, response.output
|
assert response.exit_code == 0, response.output
|
||||||
assert json.loads(response.output)["generation"] == "gen:abc"
|
assert response.stderr == ""
|
||||||
|
assert json.loads(response.stdout) == {
|
||||||
|
"changed": [],
|
||||||
|
"code": "ok",
|
||||||
|
"counts": {"changed": 0, "chunks": 0, "documents": 0, "removed": 0, "unchanged": 0},
|
||||||
|
"generation": "gen:abc",
|
||||||
|
"manifest_id": "manifest-1",
|
||||||
|
"operation": "preprocess_evidence",
|
||||||
|
"published": True,
|
||||||
|
"removed": [],
|
||||||
|
"resumed_from": None,
|
||||||
|
"run_id": "a" * 32,
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "succeeded",
|
||||||
|
"unchanged": [],
|
||||||
|
"workspaceId": "psd-clinical",
|
||||||
|
"workspaceRevision": "a" * 40,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_failure_is_structured_and_nonzero(monkeypatch, tmp_path):
|
def test_preprocess_failure_is_structured_and_nonzero(monkeypatch, tmp_path):
|
||||||
import tht.cli.preprocess_cmd as command
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
monkeypatch.setattr(command, "run_from_config", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("secret detail")))
|
config = _runtime_config(tmp_path)
|
||||||
response = CliRunner().invoke(
|
monkeypatch.setattr(
|
||||||
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
|
command,
|
||||||
|
"run_from_config",
|
||||||
|
lambda *a, **k: (_ for _ in ()).throw(RuntimeError("secret detail")),
|
||||||
)
|
)
|
||||||
|
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
|
||||||
assert response.exit_code != 0
|
assert response.exit_code != 0
|
||||||
assert json.loads(response.output) == {"status": "failed", "error": "preprocessing failed"}
|
payload = json.loads(response.stdout)
|
||||||
|
assert payload == {
|
||||||
|
"code": "preprocessing_failed",
|
||||||
|
"error": "preprocessing failed",
|
||||||
|
"operation": "preprocess_evidence",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": "psd-clinical",
|
||||||
|
"workspaceRevision": "a" * 40,
|
||||||
|
}
|
||||||
assert "secret detail" not in response.output
|
assert "secret detail" not in response.output
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_failed_job_report_is_sanitized_json_and_nonzero(monkeypatch, tmp_path):
|
def test_preprocess_failed_job_report_is_sanitized_json_and_nonzero(monkeypatch, tmp_path):
|
||||||
import tht.cli.preprocess_cmd as command
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
result = SimpleNamespace(model_dump=lambda mode=None: {
|
config = _runtime_config(tmp_path)
|
||||||
"status": "failed", "run_id": "a" * 32, "published": False,
|
result = SimpleNamespace(
|
||||||
"generation": "gen:" + "b" * 32, "changed": ["fs:one"],
|
model_dump=lambda mode=None: {
|
||||||
})
|
"status": "failed",
|
||||||
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
"run_id": "a" * 32,
|
||||||
response = CliRunner().invoke(
|
"published": False,
|
||||||
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
|
"generation": "gen:" + "b" * 32,
|
||||||
|
"changed": ["fs:one"],
|
||||||
|
"unchanged": [],
|
||||||
|
"removed": [],
|
||||||
|
"counts": {"changed": 1, "unchanged": 0, "removed": 0, "documents": 1, "chunks": 1},
|
||||||
|
"manifest_id": "manifest-1",
|
||||||
|
"resumed_from": None,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
||||||
|
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
|
||||||
assert response.exit_code == 1
|
assert response.exit_code == 1
|
||||||
payload = json.loads(response.output)
|
payload = json.loads(response.stdout)
|
||||||
assert payload["status"] == "failed"
|
assert payload["status"] == "failed"
|
||||||
assert payload["error"] == "preprocessing job failed"
|
assert payload["error"] == "preprocessing job failed"
|
||||||
|
assert payload["workspaceId"] == "psd-clinical"
|
||||||
assert "traceback" not in response.output.lower()
|
assert "traceback" not in response.output.lower()
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_real_failed_stage_result_exits_nonzero(monkeypatch, tmp_path):
|
def test_preprocess_real_failed_stage_result_exits_nonzero(monkeypatch, tmp_path):
|
||||||
import tht.cli.preprocess_cmd as command
|
|
||||||
from test_corpus_pipeline import Source, item, pipeline
|
from test_corpus_pipeline import Source, item, pipeline
|
||||||
|
|
||||||
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
|
config = _runtime_config(tmp_path)
|
||||||
result = pipeline(
|
result = pipeline(
|
||||||
tmp_path, Source([(item("one", "a"), RuntimeError("SENSITIVE EVIDENCE secret"))])
|
tmp_path,
|
||||||
|
Source([(item("one", "a"), RuntimeError("SENSITIVE EVIDENCE secret"))]),
|
||||||
).run_as_job(
|
).run_as_job(
|
||||||
workspace_id="demo", workspace_root=tmp_path,
|
workspace_id="demo",
|
||||||
|
workspace_root=tmp_path,
|
||||||
config_fingerprint="sha256:" + "1" * 64,
|
config_fingerprint="sha256:" + "1" * 64,
|
||||||
input_fingerprint="sha256:" + "2" * 64,
|
input_fingerprint="sha256:" + "2" * 64,
|
||||||
)
|
)
|
||||||
assert result.status == "failed"
|
assert result.status == "failed"
|
||||||
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
||||||
response = CliRunner().invoke(
|
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
|
||||||
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
|
|
||||||
)
|
|
||||||
assert response.exit_code == 1
|
assert response.exit_code == 1
|
||||||
assert json.loads(response.output)["status"] == "failed"
|
assert json.loads(response.stdout)["status"] == "failed"
|
||||||
assert "SENSITIVE EVIDENCE" not in response.output
|
assert "SENSITIVE EVIDENCE" not in response.output
|
||||||
assert "secret" not in response.output
|
assert "secret" not in response.output
|
||||||
|
|
||||||
@@ -75,19 +160,24 @@ def test_preprocess_real_failed_stage_result_exits_nonzero(monkeypatch, tmp_path
|
|||||||
def test_preprocess_evidence_text_uses_uncapped_result_counts(monkeypatch, tmp_path):
|
def test_preprocess_evidence_text_uses_uncapped_result_counts(monkeypatch, tmp_path):
|
||||||
import tht.cli.preprocess_cmd as command
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
result = SimpleNamespace(model_dump=lambda mode=None: {
|
config = _runtime_config(tmp_path)
|
||||||
"status": "succeeded", "run_id": "a" * 32,
|
result = SimpleNamespace(
|
||||||
"generation": "gen:" + "b" * 64, "published": True,
|
model_dump=lambda mode=None: {
|
||||||
"changed": ["fs:item"] * 100,
|
"status": "succeeded",
|
||||||
"unchanged": ["fs:item"] * 100,
|
"run_id": "a" * 32,
|
||||||
"removed": ["fs:item"] * 100,
|
"generation": "gen:" + "b" * 64,
|
||||||
"counts": {"changed": 1001, "unchanged": 902, "removed": 803},
|
"published": True,
|
||||||
})
|
"changed": ["fs:item"] * 100,
|
||||||
|
"unchanged": ["fs:item"] * 100,
|
||||||
|
"removed": ["fs:item"] * 100,
|
||||||
|
"counts": {"changed": 1001, "unchanged": 902, "removed": 803},
|
||||||
|
"manifest_id": "manifest-1",
|
||||||
|
"resumed_from": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
|
||||||
|
|
||||||
response = CliRunner().invoke(
|
response = CliRunner().invoke(app, ["preprocess", "evidence", "-c", str(config)])
|
||||||
app, ["preprocess", "evidence", "-c", str(tmp_path / "workspace.yaml")]
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.exit_code == 0, response.output
|
assert response.exit_code == 0, response.output
|
||||||
assert "changed=1001 unchanged=902 removed=803" in response.output
|
assert "changed=1001 unchanged=902 removed=803" in response.output
|
||||||
@@ -96,6 +186,7 @@ def test_preprocess_evidence_text_uses_uncapped_result_counts(monkeypatch, tmp_p
|
|||||||
def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatch, tmp_path):
|
def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatch, tmp_path):
|
||||||
import tht.cli.preprocess_cmd as command
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
|
config = _runtime_config(tmp_path)
|
||||||
called = False
|
called = False
|
||||||
|
|
||||||
def forbidden(*args, **kwargs):
|
def forbidden(*args, **kwargs):
|
||||||
@@ -106,26 +197,80 @@ def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatc
|
|||||||
response = CliRunner().invoke(
|
response = CliRunner().invoke(
|
||||||
app,
|
app,
|
||||||
[
|
[
|
||||||
"preprocess", "evidence", "--resume", "gen:" + "a" * 32,
|
"preprocess",
|
||||||
"--json", "-c", str(tmp_path / "workspace.yaml"),
|
"evidence",
|
||||||
|
"--resume",
|
||||||
|
"gen:" + "a" * 32,
|
||||||
|
"--json",
|
||||||
|
"-c",
|
||||||
|
str(config),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
assert response.exit_code != 0
|
assert response.exit_code != 0
|
||||||
assert json.loads(response.output) == {
|
assert json.loads(response.output) == {
|
||||||
"status": "failed", "error": "resume requires a preprocessing run id"
|
"code": "invalid_resume",
|
||||||
|
"error": "resume requires a preprocessing run id",
|
||||||
|
"operation": "preprocess_evidence",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
}
|
}
|
||||||
assert called is False
|
assert called is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_path):
|
||||||
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
|
config = _runtime_config(tmp_path, name="3")
|
||||||
|
calls = {}
|
||||||
|
|
||||||
|
class FakePipeline:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
store,
|
||||||
|
sources,
|
||||||
|
embedder,
|
||||||
|
vector_store,
|
||||||
|
embedding_model,
|
||||||
|
embedding_dimensions,
|
||||||
|
chunk_policy,
|
||||||
|
pipeline_version,
|
||||||
|
retain_published_generations,
|
||||||
|
):
|
||||||
|
calls["init"] = {
|
||||||
|
"embedding_model": embedding_model,
|
||||||
|
"embedding_dimensions": embedding_dimensions,
|
||||||
|
"pipeline_version": pipeline_version,
|
||||||
|
}
|
||||||
|
|
||||||
|
def run_as_job(self, **kwargs):
|
||||||
|
calls["run_as_job"] = kwargs
|
||||||
|
return SimpleNamespace(model_dump=lambda mode=None: {"status": "succeeded"})
|
||||||
|
|
||||||
|
monkeypatch.setattr("tht.adapters.factory.build_evidence_sources", lambda cfg: [])
|
||||||
|
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: object())
|
||||||
|
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda cfg: object())
|
||||||
|
monkeypatch.setattr("tht.corpus.pipeline.CorpusPipeline", FakePipeline)
|
||||||
|
|
||||||
|
command.run_from_config(config)
|
||||||
|
|
||||||
|
assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
|
||||||
|
assert calls["run_as_job"]["input_fingerprint"] != calls["run_as_job"]["config_fingerprint"]
|
||||||
|
|
||||||
|
|
||||||
def test_preprocess_evidence_gc_json_is_pristine(monkeypatch, tmp_path):
|
def test_preprocess_evidence_gc_json_is_pristine(monkeypatch, tmp_path):
|
||||||
import tht.cli.preprocess_cmd as command
|
import tht.cli.preprocess_cmd as command
|
||||||
|
|
||||||
|
config = _runtime_config(tmp_path)
|
||||||
monkeypatch.setattr(command, "gc_from_config", lambda *a, **k: {
|
monkeypatch.setattr(command, "gc_from_config", lambda *a, **k: {
|
||||||
"status": "succeeded", "dry_run": True, "evicted": [], "failures": [],
|
"status": "succeeded",
|
||||||
|
"dry_run": True,
|
||||||
|
"evicted": [],
|
||||||
|
"failures": [],
|
||||||
})
|
})
|
||||||
response = CliRunner().invoke(
|
response = CliRunner().invoke(
|
||||||
app, ["preprocess", "evidence", "gc", "--dry-run", "--json", "-c",
|
app,
|
||||||
str(tmp_path / "workspace.yaml")]
|
["preprocess", "evidence", "gc", "--dry-run", "--json", "-c", str(config)],
|
||||||
)
|
)
|
||||||
assert response.exit_code == 0, response.output
|
assert response.exit_code == 0, response.output
|
||||||
assert json.loads(response.output)["dry_run"] is True
|
assert json.loads(response.output)["dry_run"] is True
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -9,6 +10,7 @@ from typer.testing import CliRunner
|
|||||||
|
|
||||||
from tht.cli import app
|
from tht.cli import app
|
||||||
from tht.memory import MemoryRecord, save_registry
|
from tht.memory import MemoryRecord, save_registry
|
||||||
|
from tht.ports.vector import VectorStoreError
|
||||||
|
|
||||||
|
|
||||||
class _FakeEmbedder:
|
class _FakeEmbedder:
|
||||||
@@ -33,6 +35,10 @@ class _FakeVectorStore:
|
|||||||
return 3
|
return 3
|
||||||
|
|
||||||
|
|
||||||
|
def _sha_file(path: Path) -> str:
|
||||||
|
return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def _qdrant_runtime_config(tmp_path: Path) -> Path:
|
def _qdrant_runtime_config(tmp_path: Path) -> Path:
|
||||||
cfg = tmp_path / "workspace.yaml"
|
cfg = tmp_path / "workspace.yaml"
|
||||||
cfg.write_text(
|
cfg.write_text(
|
||||||
@@ -76,9 +82,7 @@ tables:
|
|||||||
type: bigint
|
type: bigint
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
(tmp_path / "artifacts" / "mschema" / "annotations.yaml").write_text(
|
(tmp_path / "artifacts" / "mschema" / "annotations.yaml").write_text("tables: {}\n")
|
||||||
"tables: {}\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _memory_record() -> MemoryRecord:
|
def _memory_record() -> MemoryRecord:
|
||||||
@@ -111,6 +115,73 @@ def test_vector_index_schema_accepts_qdrant_only_runtime_config(tmp_path, monkey
|
|||||||
assert store.upserts
|
assert store.upserts
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_index_schema_json_is_pristine_and_reports_artifacts(tmp_path, monkeypatch):
|
||||||
|
cfg = _qdrant_runtime_config(tmp_path)
|
||||||
|
_write_schema_artifacts(tmp_path)
|
||||||
|
store = _FakeVectorStore()
|
||||||
|
|
||||||
|
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: store)
|
||||||
|
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda _: _FakeEmbedder())
|
||||||
|
|
||||||
|
response = CliRunner().invoke(app, ["vector", "index-schema", "--json", "-c", str(cfg)])
|
||||||
|
|
||||||
|
assert response.exit_code == 0, response.output
|
||||||
|
assert response.stderr == ""
|
||||||
|
assert json.loads(response.stdout) == {
|
||||||
|
"artifactIdentities": [
|
||||||
|
{
|
||||||
|
"digest": _sha_file(tmp_path / "artifacts" / "mschema" / "annotations.yaml"),
|
||||||
|
"kind": "schema_annotations",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"digest": _sha_file(tmp_path / "artifacts" / "mschema" / "physical.yaml"),
|
||||||
|
"kind": "physical_schema",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"code": "ok",
|
||||||
|
"collection": "psd-clinical",
|
||||||
|
"counts": {
|
||||||
|
"added": 2,
|
||||||
|
"columns": 1,
|
||||||
|
"deleted": 0,
|
||||||
|
"records": 2,
|
||||||
|
"tables": 1,
|
||||||
|
"unchanged": 0,
|
||||||
|
"updated": 0,
|
||||||
|
},
|
||||||
|
"operation": "index_schema",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "succeeded",
|
||||||
|
"workspaceId": "psd-clinical",
|
||||||
|
"workspaceRevision": "a" * 40,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_index_schema_json_failure_is_pristine(tmp_path, monkeypatch):
|
||||||
|
cfg = _qdrant_runtime_config(tmp_path)
|
||||||
|
_write_schema_artifacts(tmp_path)
|
||||||
|
|
||||||
|
def boom(cfg, require_write):
|
||||||
|
raise VectorStoreError("semantic_index_incompatible")
|
||||||
|
|
||||||
|
monkeypatch.setattr("tht.adapters.factory.build_vector_store", boom)
|
||||||
|
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda _: _FakeEmbedder())
|
||||||
|
|
||||||
|
response = CliRunner().invoke(app, ["vector", "index-schema", "--json", "-c", str(cfg)])
|
||||||
|
|
||||||
|
assert response.exit_code == 1
|
||||||
|
assert response.stderr == ""
|
||||||
|
assert json.loads(response.stdout) == {
|
||||||
|
"code": "semantic_index_incompatible",
|
||||||
|
"error": "semantic index incompatible",
|
||||||
|
"operation": "index_schema",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": "psd-clinical",
|
||||||
|
"workspaceRevision": "a" * 40,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_memory_promote_accepts_qdrant_only_runtime_config(tmp_path, monkeypatch):
|
def test_memory_promote_accepts_qdrant_only_runtime_config(tmp_path, monkeypatch):
|
||||||
cfg = _qdrant_runtime_config(tmp_path)
|
cfg = _qdrant_runtime_config(tmp_path)
|
||||||
store = _FakeVectorStore()
|
store = _FakeVectorStore()
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ class FakeQdrantHttp:
|
|||||||
self.malformed_query = False
|
self.malformed_query = False
|
||||||
self.malformed_scroll = False
|
self.malformed_scroll = False
|
||||||
self.scroll_pages: list[dict] | None = None
|
self.scroll_pages: list[dict] | None = None
|
||||||
|
self.drop_collection_on_points = False
|
||||||
|
|
||||||
def request(self, method, url, *, json=None, timeout=None):
|
def request(self, method, url, *, json=None, timeout=None):
|
||||||
self.calls.append((method, url, json))
|
self.calls.append((method, url, json))
|
||||||
@@ -75,6 +76,9 @@ class FakeQdrantHttp:
|
|||||||
return FakeResponse(200, {"status": "ok"})
|
return FakeResponse(200, {"status": "ok"})
|
||||||
|
|
||||||
if method == "PUT" and path == "/collections/workspace-semantic/points":
|
if method == "PUT" and path == "/collections/workspace-semantic/points":
|
||||||
|
if self.drop_collection_on_points:
|
||||||
|
self.collection = None
|
||||||
|
return FakeResponse(404, {"status": "error"})
|
||||||
for point in json["points"]:
|
for point in json["points"]:
|
||||||
self.points[point["id"]] = point
|
self.points[point["id"]] = point
|
||||||
return FakeResponse(200, {"result": {"status": "acknowledged"}})
|
return FakeResponse(200, {"result": {"status": "acknowledged"}})
|
||||||
@@ -161,13 +165,16 @@ def _write_record(record_id: str, kind: str, *, metadata=None):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _store(fake: FakeQdrantHttp) -> QdrantVectorStore:
|
def _store(
|
||||||
|
fake: FakeQdrantHttp, *, collection_lifecycle: str = "self_heal"
|
||||||
|
) -> QdrantVectorStore:
|
||||||
return QdrantVectorStore(
|
return QdrantVectorStore(
|
||||||
base_url="http://qdrant:6333",
|
base_url="http://qdrant:6333",
|
||||||
collection="workspace-semantic",
|
collection="workspace-semantic",
|
||||||
workspace_id="demo",
|
workspace_id="demo",
|
||||||
workspace_revision="a" * 40,
|
workspace_revision="a" * 40,
|
||||||
expected_dimension=1024,
|
expected_dimension=1024,
|
||||||
|
collection_lifecycle=collection_lifecycle,
|
||||||
request=fake.request,
|
request=fake.request,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -212,6 +219,87 @@ def test_upsert_refuses_collection_dimension_or_distance_mismatch_without_recrea
|
|||||||
assert creates == []
|
assert creates == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_require_existing_refuses_missing_collection_without_creating():
|
||||||
|
fake = FakeQdrantHttp()
|
||||||
|
store = _store(fake, collection_lifecycle="require_existing")
|
||||||
|
|
||||||
|
with pytest.raises(VectorStoreError, match="semantic_index_incompatible"):
|
||||||
|
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||||
|
|
||||||
|
assert fake.collection is None
|
||||||
|
creates = [
|
||||||
|
call
|
||||||
|
for call in fake.calls
|
||||||
|
if call[0] == "PUT" and call[1].endswith("/collections/workspace-semantic")
|
||||||
|
]
|
||||||
|
assert creates == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_require_existing_refuses_incompatible_collection_without_mutating():
|
||||||
|
fake = FakeQdrantHttp(dimension=384, distance="Dot")
|
||||||
|
fake.collection = {"vectors": {"size": 384, "distance": "Dot"}}
|
||||||
|
store = _store(fake, collection_lifecycle="require_existing")
|
||||||
|
|
||||||
|
with pytest.raises(VectorStoreError, match="semantic_index_incompatible"):
|
||||||
|
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||||
|
|
||||||
|
assert fake.payload_indexes == set()
|
||||||
|
mutating = [call for call in fake.calls if call[0] == "PUT"]
|
||||||
|
assert mutating == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_require_existing_writes_to_existing_compatible_collection():
|
||||||
|
fake = FakeQdrantHttp()
|
||||||
|
fake.collection = {"vectors": {"size": 1024, "distance": "Cosine"}}
|
||||||
|
fake.payload_indexes = {
|
||||||
|
"content_hash",
|
||||||
|
"document_id",
|
||||||
|
"kind",
|
||||||
|
"record_key",
|
||||||
|
"record_kind",
|
||||||
|
"vector_generation",
|
||||||
|
"workspace_id",
|
||||||
|
"workspace_revision",
|
||||||
|
}
|
||||||
|
store = _store(fake, collection_lifecycle="require_existing")
|
||||||
|
|
||||||
|
assert store.upsert("memory", [_write_record("memory:1", "memory")]) == 1
|
||||||
|
|
||||||
|
create_or_index = [
|
||||||
|
call
|
||||||
|
for call in fake.calls
|
||||||
|
if call[0] == "PUT" and not call[1].endswith("/points?wait=true")
|
||||||
|
]
|
||||||
|
assert create_or_index == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_require_existing_fails_if_collection_disappears_after_preflight():
|
||||||
|
fake = FakeQdrantHttp()
|
||||||
|
fake.collection = {"vectors": {"size": 1024, "distance": "Cosine"}}
|
||||||
|
fake.payload_indexes = {
|
||||||
|
"content_hash",
|
||||||
|
"document_id",
|
||||||
|
"kind",
|
||||||
|
"record_key",
|
||||||
|
"record_kind",
|
||||||
|
"vector_generation",
|
||||||
|
"workspace_id",
|
||||||
|
"workspace_revision",
|
||||||
|
}
|
||||||
|
fake.drop_collection_on_points = True
|
||||||
|
store = _store(fake, collection_lifecycle="require_existing")
|
||||||
|
|
||||||
|
with pytest.raises(VectorStoreError, match="HTTP 404"):
|
||||||
|
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||||
|
|
||||||
|
creates = [
|
||||||
|
call
|
||||||
|
for call in fake.calls
|
||||||
|
if call[0] == "PUT" and call[1].endswith("/collections/workspace-semantic")
|
||||||
|
]
|
||||||
|
assert creates == []
|
||||||
|
|
||||||
|
|
||||||
def test_health_fails_when_the_bound_collection_is_missing():
|
def test_health_fails_when_the_bound_collection_is_missing():
|
||||||
fake = FakeQdrantHttp()
|
fake = FakeQdrantHttp()
|
||||||
|
|
||||||
|
|||||||
@@ -113,6 +113,61 @@ def test_signed_http_file_resolves_in_memory_and_preserves_provenance_order(tmp_
|
|||||||
assert_no_canaries(repr(adapter))
|
assert_no_canaries(repr(adapter))
|
||||||
|
|
||||||
|
|
||||||
|
def test_qdrant_runtime_config_keeps_require_existing_and_http_policy_fields(tmp_path):
|
||||||
|
path = tmp_path / "runtime.yaml"
|
||||||
|
path.write_text(
|
||||||
|
yaml.safe_dump(
|
||||||
|
{
|
||||||
|
"runtime_identity": {
|
||||||
|
"workspace_id": "psd-clinical",
|
||||||
|
"workspace_revision": "a" * 40,
|
||||||
|
},
|
||||||
|
"dwh": {
|
||||||
|
"type": "postgres_direct",
|
||||||
|
"connection": {
|
||||||
|
"database": "analytics",
|
||||||
|
"schema": "public",
|
||||||
|
"user": "reader",
|
||||||
|
"password": "secret",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"vectors": {
|
||||||
|
"type": "qdrant",
|
||||||
|
"base_url": "http://qdrant:6333",
|
||||||
|
"collection": "psd-clinical",
|
||||||
|
"collection_lifecycle": "require_existing",
|
||||||
|
},
|
||||||
|
"embeddings": {
|
||||||
|
"provider": "ollama_internal",
|
||||||
|
"base_url": "http://embedding:11434",
|
||||||
|
"model": "qwen3-embedding:0.6b",
|
||||||
|
"dim": 1024,
|
||||||
|
},
|
||||||
|
"evidence": {
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"type": "http",
|
||||||
|
"urls": ["https://evidence.example.test/guide.md"],
|
||||||
|
"allow_private_hosts": True,
|
||||||
|
"max_redirects": 2,
|
||||||
|
"max_cache_bytes": 1234,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg = load_config(path)
|
||||||
|
rendered = cfg.model_dump(mode="json")
|
||||||
|
|
||||||
|
assert cfg.vectors.collection_lifecycle == "require_existing"
|
||||||
|
assert rendered["vectors"]["collection_lifecycle"] == "require_existing"
|
||||||
|
assert rendered["evidence"]["sources"][0]["allow_private_hosts"] is True
|
||||||
|
assert rendered["evidence"]["sources"][0]["max_redirects"] == 2
|
||||||
|
assert rendered["evidence"]["sources"][0]["max_cache_bytes"] == 1234
|
||||||
|
|
||||||
|
|
||||||
def test_signed_http_file_requires_explicit_provenance_urls(tmp_path):
|
def test_signed_http_file_requires_explicit_provenance_urls(tmp_path):
|
||||||
secret_file = tmp_path / "signed-urls.json"
|
secret_file = tmp_path / "signed-urls.json"
|
||||||
secret_file.write_text(json.dumps([
|
secret_file.write_text(json.dumps([
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
from datetime import datetime
|
import hashlib
|
||||||
|
import json
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
from typer.testing import CliRunner
|
from typer.testing import CliRunner
|
||||||
@@ -15,10 +17,19 @@ from tht.mschema.models import (
|
|||||||
)
|
)
|
||||||
from tht.mschema.render import to_mschema_text, to_schema_dict
|
from tht.mschema.render import to_mschema_text, to_schema_dict
|
||||||
|
|
||||||
|
RUNNER = CliRunner()
|
||||||
|
|
||||||
|
|
||||||
|
def _json_sha(value) -> str:
|
||||||
|
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
|
return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def _physical():
|
def _physical():
|
||||||
return PhysicalSchema(
|
return PhysicalSchema(
|
||||||
database="d", schema="s", introspected_at=datetime(2026, 1, 1),
|
database="d",
|
||||||
|
schema="s",
|
||||||
|
introspected_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||||
tables={
|
tables={
|
||||||
"dim_patient": TablePhysical(
|
"dim_patient": TablePhysical(
|
||||||
columns={"cod_paz": ColumnPhysical(type="bigint", pk=True)},
|
columns={"cod_paz": ColumnPhysical(type="bigint", pk=True)},
|
||||||
@@ -42,10 +53,16 @@ def _annotations_with_fks():
|
|||||||
tables={
|
tables={
|
||||||
"fact_ablazione": TableAnnotation(
|
"fact_ablazione": TableAnnotation(
|
||||||
foreign_keys=[
|
foreign_keys=[
|
||||||
ForeignKey(columns=["cod_paz"], ref_table="dim_patient",
|
ForeignKey(
|
||||||
ref_columns=["cod_paz"]),
|
columns=["cod_paz"],
|
||||||
ForeignKey(columns=["data_time_key"], ref_table="dim_time",
|
ref_table="dim_patient",
|
||||||
ref_columns=["day_key"]),
|
ref_columns=["cod_paz"],
|
||||||
|
),
|
||||||
|
ForeignKey(
|
||||||
|
columns=["data_time_key"],
|
||||||
|
ref_table="dim_time",
|
||||||
|
ref_columns=["day_key"],
|
||||||
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -61,8 +78,11 @@ def test_mschema_text_renders_annotation_fks():
|
|||||||
def test_schema_dict_merges_annotation_fks():
|
def test_schema_dict_merges_annotation_fks():
|
||||||
d = to_schema_dict(_physical(), _annotations_with_fks())
|
d = to_schema_dict(_physical(), _annotations_with_fks())
|
||||||
fks = d["fact_ablazione"]["foreign_keys"]
|
fks = d["fact_ablazione"]["foreign_keys"]
|
||||||
assert {"columns": ["cod_paz"], "ref_table": "dim_patient",
|
assert {
|
||||||
"ref_columns": ["cod_paz"]} in fks
|
"columns": ["cod_paz"],
|
||||||
|
"ref_table": "dim_patient",
|
||||||
|
"ref_columns": ["cod_paz"],
|
||||||
|
} in fks
|
||||||
|
|
||||||
|
|
||||||
def test_find_orphans_flags_broken_annotation_fk():
|
def test_find_orphans_flags_broken_annotation_fk():
|
||||||
@@ -70,10 +90,16 @@ def test_find_orphans_flags_broken_annotation_fk():
|
|||||||
tables={
|
tables={
|
||||||
"fact_ablazione": TableAnnotation(
|
"fact_ablazione": TableAnnotation(
|
||||||
foreign_keys=[
|
foreign_keys=[
|
||||||
ForeignKey(columns=["cod_paz"], ref_table="dim_sparita",
|
ForeignKey(
|
||||||
ref_columns=["x"]),
|
columns=["cod_paz"],
|
||||||
ForeignKey(columns=["colonna_sparita"], ref_table="dim_time",
|
ref_table="dim_sparita",
|
||||||
ref_columns=["day_key"]),
|
ref_columns=["x"],
|
||||||
|
),
|
||||||
|
ForeignKey(
|
||||||
|
columns=["colonna_sparita"],
|
||||||
|
ref_table="dim_time",
|
||||||
|
ref_columns=["day_key"],
|
||||||
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -91,22 +117,139 @@ def _write_workspace(tmp_path):
|
|||||||
_physical().to_yaml(tmp_path / "artifacts" / "mschema" / "physical.yaml")
|
_physical().to_yaml(tmp_path / "artifacts" / "mschema" / "physical.yaml")
|
||||||
cfg = tmp_path / "workspace.yaml"
|
cfg = tmp_path / "workspace.yaml"
|
||||||
cfg.write_text(
|
cfg.write_text(
|
||||||
"database: {database: d, schema: s, user: u, password: p, transport: direct}\n"
|
f"""
|
||||||
f"paths: {{artifacts: {tmp_path/'artifacts'}, indexes: {tmp_path/'i'}, sessions: {tmp_path/'s'}}}\n"
|
runtime_identity:
|
||||||
|
workspace_id: demo
|
||||||
|
workspace_revision: {'a' * 40}
|
||||||
|
database: {{database: d, schema: s, user: u, password: p, transport: direct}}
|
||||||
|
paths: {{artifacts: {tmp_path / 'artifacts'}, indexes: {tmp_path / 'i'}, sessions: {tmp_path / 's'}}}
|
||||||
|
"""
|
||||||
)
|
)
|
||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
def test_suggest_fks_prints_candidates(tmp_path):
|
def test_suggest_fks_prints_candidates(tmp_path):
|
||||||
cfg = _write_workspace(tmp_path)
|
cfg = _write_workspace(tmp_path)
|
||||||
res = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg)])
|
res = RUNNER.invoke(app, ["schema", "suggest-fks", "-c", str(cfg)])
|
||||||
assert res.exit_code == 0, res.output
|
assert res.exit_code == 0, res.output
|
||||||
data = yaml.safe_load(res.output.rsplit("\n", 2)[0].split("FK candidate")[0])
|
data = yaml.safe_load(res.output.rsplit("\n", 2)[0].split("FK candidate")[0])
|
||||||
fks = data["tables"]["fact_ablazione"]["foreign_keys"]
|
fks = data["tables"]["fact_ablazione"]["foreign_keys"]
|
||||||
assert {"columns": ["cod_paz"], "ref_table": "dim_patient",
|
assert {
|
||||||
"ref_columns": ["cod_paz"]} in fks
|
"columns": ["cod_paz"],
|
||||||
assert {"columns": ["data_time_key"], "ref_table": "dim_time",
|
"ref_table": "dim_patient",
|
||||||
"ref_columns": ["day_key"]} in fks
|
"ref_columns": ["cod_paz"],
|
||||||
|
} in fks
|
||||||
|
assert {
|
||||||
|
"columns": ["data_time_key"],
|
||||||
|
"ref_table": "dim_time",
|
||||||
|
"ref_columns": ["day_key"],
|
||||||
|
} in fks
|
||||||
|
|
||||||
|
|
||||||
|
def test_suggest_fks_json_is_pristine_and_stable(tmp_path):
|
||||||
|
cfg = _write_workspace(tmp_path)
|
||||||
|
first = tmp_path / "second.sql"
|
||||||
|
first.write_text(
|
||||||
|
"SELECT f.esito FROM datawarehouse.fact_ablazione f "
|
||||||
|
"JOIN datawarehouse.dim_patient p ON f.cod_paz = p.cod_paz"
|
||||||
|
)
|
||||||
|
second = tmp_path / "first.sql"
|
||||||
|
second.write_text(
|
||||||
|
"SELECT dt.year FROM datawarehouse.fact_ablazione f "
|
||||||
|
"JOIN datawarehouse.dim_time dt ON f.data_time_key = dt.day_key"
|
||||||
|
)
|
||||||
|
|
||||||
|
response = RUNNER.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"schema",
|
||||||
|
"suggest-fks",
|
||||||
|
"-c",
|
||||||
|
str(cfg),
|
||||||
|
"--from-sql",
|
||||||
|
str(first),
|
||||||
|
"--from-sql",
|
||||||
|
str(second),
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.exit_code == 0, response.output
|
||||||
|
assert response.stderr == ""
|
||||||
|
payload = json.loads(response.stdout)
|
||||||
|
assert payload["schemaVersion"] == 1
|
||||||
|
assert payload["status"] == "succeeded"
|
||||||
|
assert payload["code"] == "ok"
|
||||||
|
assert payload["operation"] == "schema_suggest_fks"
|
||||||
|
assert payload["workspaceId"] == "demo"
|
||||||
|
assert payload["workspaceRevision"] == "a" * 40
|
||||||
|
assert payload["counts"] == {
|
||||||
|
"ambiguousColumns": 0,
|
||||||
|
"candidateTables": 1,
|
||||||
|
"candidates": 2,
|
||||||
|
"minedJoins": 2,
|
||||||
|
"sqlFiles": 2,
|
||||||
|
}
|
||||||
|
assert payload["candidateDocument"] == {
|
||||||
|
"annotations": {
|
||||||
|
"tables": {
|
||||||
|
"fact_ablazione": {
|
||||||
|
"foreign_keys": [
|
||||||
|
{
|
||||||
|
"columns": ["cod_paz"],
|
||||||
|
"ref_columns": ["cod_paz"],
|
||||||
|
"ref_table": "dim_patient",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"columns": ["data_time_key"],
|
||||||
|
"ref_columns": ["day_key"],
|
||||||
|
"ref_table": "dim_time",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"counts": {"candidateTables": 1, "candidates": 2},
|
||||||
|
"schemaVersion": 1,
|
||||||
|
}
|
||||||
|
assert payload["candidate_count"] == payload["counts"]["candidates"]
|
||||||
|
candidate_yaml = payload["candidate_yaml"]
|
||||||
|
assert payload["candidateDigest"] == "sha256:" + hashlib.sha256(candidate_yaml.encode("utf-8")).hexdigest()
|
||||||
|
assert yaml.safe_load(candidate_yaml) == payload["candidateDocument"]["annotations"]
|
||||||
|
|
||||||
|
rerun = RUNNER.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"schema",
|
||||||
|
"suggest-fks",
|
||||||
|
"-c",
|
||||||
|
str(cfg),
|
||||||
|
"--from-sql",
|
||||||
|
str(second),
|
||||||
|
"--from-sql",
|
||||||
|
str(first),
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert rerun.exit_code == 0, rerun.output
|
||||||
|
assert json.loads(rerun.stdout) == payload
|
||||||
|
|
||||||
|
|
||||||
|
def test_suggest_fks_json_rejects_invalid_assume_without_prose(tmp_path):
|
||||||
|
cfg = _write_workspace(tmp_path)
|
||||||
|
|
||||||
|
response = RUNNER.invoke(
|
||||||
|
app,
|
||||||
|
["schema", "suggest-fks", "-c", str(cfg), "--assume", "cod_x=nope", "--json"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.exit_code == 1
|
||||||
|
assert response.stderr == ""
|
||||||
|
payload = json.loads(response.stdout)
|
||||||
|
assert payload["schemaVersion"] == 1
|
||||||
|
assert payload["status"] == "failed"
|
||||||
|
assert payload["code"] == "invalid_argument"
|
||||||
|
assert payload["error"] == "invalid assume mapping"
|
||||||
|
|
||||||
|
|
||||||
def test_mine_join_pairs_from_approved_sql():
|
def test_mine_join_pairs_from_approved_sql():
|
||||||
@@ -122,23 +265,22 @@ def test_mine_join_pairs_from_approved_sql():
|
|||||||
"""
|
"""
|
||||||
pairs = mine_join_pairs(sql, _physical())
|
pairs = mine_join_pairs(sql, _physical())
|
||||||
assert pairs[("fact_ablazione", "data_time_key", "dim_time", "day_key")] == 1
|
assert pairs[("fact_ablazione", "data_time_key", "dim_time", "day_key")] == 1
|
||||||
# il join CTE-CTE (abl.year=b.year) non produce coppie
|
|
||||||
assert len(pairs) == 1
|
assert len(pairs) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_mine_join_pairs_ignores_non_pk_pairs_and_bad_sql():
|
def test_mine_join_pairs_ignores_non_pk_pairs_and_bad_sql():
|
||||||
from tht.mschema.fkmine import mine_join_pairs
|
from tht.mschema.fkmine import mine_join_pairs
|
||||||
|
|
||||||
# esito=esito: nessun lato e' PK -> scartato
|
sql = "SELECT * FROM fact_ablazione a JOIN fact_ablazione b ON a.esito = b.esito"
|
||||||
sql = ("SELECT * FROM fact_ablazione a JOIN fact_ablazione b "
|
|
||||||
"ON a.esito = b.esito")
|
|
||||||
assert len(mine_join_pairs(sql, _physical())) == 0
|
assert len(mine_join_pairs(sql, _physical())) == 0
|
||||||
assert len(mine_join_pairs("WITH broken (", _physical())) == 0
|
assert len(mine_join_pairs("WITH broken (", _physical())) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_suggest_fks_skips_generic_and_ambiguous_pks(tmp_path):
|
def test_suggest_fks_skips_generic_and_ambiguous_pks(tmp_path):
|
||||||
phys = PhysicalSchema(
|
phys = PhysicalSchema(
|
||||||
database="d", schema="s", introspected_at=datetime(2026, 1, 1),
|
database="d",
|
||||||
|
schema="s",
|
||||||
|
introspected_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||||
tables={
|
tables={
|
||||||
"dim_a": TablePhysical(columns={"id": ColumnPhysical(type="int", pk=True)}),
|
"dim_a": TablePhysical(columns={"id": ColumnPhysical(type="int", pk=True)}),
|
||||||
"dim_b": TablePhysical(columns={"id": ColumnPhysical(type="int", pk=True)}),
|
"dim_b": TablePhysical(columns={"id": ColumnPhysical(type="int", pk=True)}),
|
||||||
@@ -158,14 +300,14 @@ def test_suggest_fks_skips_generic_and_ambiguous_pks(tmp_path):
|
|||||||
"database: {database: d, schema: s, user: u, password: p, transport: direct}\n"
|
"database: {database: d, schema: s, user: u, password: p, transport: direct}\n"
|
||||||
f"paths: {{artifacts: {tmp_path/'artifacts'}, indexes: {tmp_path/'i'}, sessions: {tmp_path/'s'}}}\n"
|
f"paths: {{artifacts: {tmp_path/'artifacts'}, indexes: {tmp_path/'i'}, sessions: {tmp_path/'s'}}}\n"
|
||||||
)
|
)
|
||||||
res = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg)])
|
res = RUNNER.invoke(app, ["schema", "suggest-fks", "-c", str(cfg)])
|
||||||
assert res.exit_code == 0, res.output
|
assert res.exit_code == 0, res.output
|
||||||
assert "nessuna FK da suggerire" in res.output # id generico, cod_x ambigua
|
assert "nessuna FK da suggerire" in res.output
|
||||||
assert "cod_x" in res.output # segnalata come ambigua saltata
|
assert "cod_x" in res.output
|
||||||
|
|
||||||
# --assume disambigua la PK multi-proprietario
|
res2 = RUNNER.invoke(
|
||||||
res2 = CliRunner().invoke(
|
app,
|
||||||
app, ["schema", "suggest-fks", "-c", str(cfg), "--assume", "cod_x=dim_c1"]
|
["schema", "suggest-fks", "-c", str(cfg), "--assume", "cod_x=dim_c1"],
|
||||||
)
|
)
|
||||||
assert res2.exit_code == 0, res2.output
|
assert res2.exit_code == 0, res2.output
|
||||||
yaml_text = "\n".join(
|
yaml_text = "\n".join(
|
||||||
@@ -173,16 +315,19 @@ def test_suggest_fks_skips_generic_and_ambiguous_pks(tmp_path):
|
|||||||
)
|
)
|
||||||
data = yaml.safe_load(yaml_text)
|
data = yaml.safe_load(yaml_text)
|
||||||
fact_fks = data["tables"]["fact_f"]["foreign_keys"]
|
fact_fks = data["tables"]["fact_f"]["foreign_keys"]
|
||||||
assert {"columns": ["cod_x"], "ref_table": "dim_c1",
|
assert {
|
||||||
"ref_columns": ["cod_x"]} in fact_fks
|
"columns": ["cod_x"],
|
||||||
# dim_c2.cod_x -> dim_c1 (estensione 1:1), ma NON dim_c1 -> se stessa
|
"ref_table": "dim_c1",
|
||||||
|
"ref_columns": ["cod_x"],
|
||||||
|
} in fact_fks
|
||||||
assert "dim_c1" not in data["tables"] or all(
|
assert "dim_c1" not in data["tables"] or all(
|
||||||
fk["ref_table"] != "dim_c1" for fk in data["tables"].get("dim_c1", {}).get("foreign_keys", [])
|
fk["ref_table"] != "dim_c1"
|
||||||
|
for fk in data["tables"].get("dim_c1", {}).get("foreign_keys", [])
|
||||||
)
|
)
|
||||||
|
|
||||||
# --assume con tabella inesistente -> errore chiaro
|
res3 = RUNNER.invoke(
|
||||||
res3 = CliRunner().invoke(
|
app,
|
||||||
app, ["schema", "suggest-fks", "-c", str(cfg), "--assume", "cod_x=nope"]
|
["schema", "suggest-fks", "-c", str(cfg), "--assume", "cod_x=nope"],
|
||||||
)
|
)
|
||||||
assert res3.exit_code == 1
|
assert res3.exit_code == 1
|
||||||
assert "non valido" in res3.output
|
assert "non valido" in res3.output
|
||||||
@@ -190,20 +335,122 @@ def test_suggest_fks_skips_generic_and_ambiguous_pks(tmp_path):
|
|||||||
|
|
||||||
def test_suggest_fks_from_sql_mines_joins(tmp_path):
|
def test_suggest_fks_from_sql_mines_joins(tmp_path):
|
||||||
cfg = _write_workspace(tmp_path)
|
cfg = _write_workspace(tmp_path)
|
||||||
sqldir = tmp_path / "approved"
|
sql_file = tmp_path / "approved.sql"
|
||||||
sqldir.mkdir()
|
sql_file.write_text(
|
||||||
(sqldir / "q1.sql").write_text(
|
|
||||||
"SELECT f.esito FROM datawarehouse.fact_ablazione f "
|
"SELECT f.esito FROM datawarehouse.fact_ablazione f "
|
||||||
"JOIN datawarehouse.dim_patient p ON f.cod_paz = p.cod_paz"
|
"JOIN datawarehouse.dim_patient p ON f.cod_paz = p.cod_paz"
|
||||||
)
|
)
|
||||||
res = CliRunner().invoke(
|
res = RUNNER.invoke(
|
||||||
app, ["schema", "suggest-fks", "-c", str(cfg), "--from-sql", str(sqldir)]
|
app,
|
||||||
|
["schema", "suggest-fks", "-c", str(cfg), "--from-sql", str(sql_file)],
|
||||||
)
|
)
|
||||||
assert res.exit_code == 0, res.output
|
assert res.exit_code == 0, res.output
|
||||||
assert "Minati 1 equi-join da 1 file SQL" in res.output
|
assert "Minati 1 equi-join da 1 file SQL" in res.output
|
||||||
assert "ref_table: dim_patient" in res.output
|
assert "ref_table: dim_patient" in res.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_check_json_validates_staged_annotations_without_mutating_runtime(tmp_path):
|
||||||
|
cfg = _write_workspace(tmp_path)
|
||||||
|
runtime_annotations = tmp_path / "artifacts" / "mschema" / "annotations.yaml"
|
||||||
|
runtime_annotations.write_text("tables: {}\n")
|
||||||
|
reviewed = tmp_path / "reviewed.yaml"
|
||||||
|
_annotations_with_fks().to_yaml(reviewed)
|
||||||
|
|
||||||
|
response = RUNNER.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"schema",
|
||||||
|
"check",
|
||||||
|
"-c",
|
||||||
|
str(cfg),
|
||||||
|
"--annotations",
|
||||||
|
str(reviewed),
|
||||||
|
"--reviewed-candidates",
|
||||||
|
"sha256:" + "b" * 64,
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.exit_code == 0, response.output
|
||||||
|
assert response.stderr == ""
|
||||||
|
payload = json.loads(response.stdout)
|
||||||
|
assert payload["orphan_count"] == 0
|
||||||
|
assert payload["reviewed_candidates_digest"] == "sha256:" + "b" * 64
|
||||||
|
assert payload["annotations_digest"] == "sha256:" + hashlib.sha256(reviewed.read_bytes()).hexdigest()
|
||||||
|
assert payload == {
|
||||||
|
"annotationsDigest": _json_sha(
|
||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"tables": {
|
||||||
|
"fact_ablazione": {
|
||||||
|
"foreign_keys": [
|
||||||
|
{
|
||||||
|
"columns": ["cod_paz"],
|
||||||
|
"ref_columns": ["cod_paz"],
|
||||||
|
"ref_table": "dim_patient",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"columns": ["data_time_key"],
|
||||||
|
"ref_columns": ["day_key"],
|
||||||
|
"ref_table": "dim_time",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"schemaVersion": 1,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"annotations_digest": "sha256:" + hashlib.sha256(reviewed.read_bytes()).hexdigest(),
|
||||||
|
"code": "ok",
|
||||||
|
"orphan_count": 0,
|
||||||
|
"reviewed_candidates_digest": "sha256:" + "b" * 64,
|
||||||
|
"counts": {"annotationTables": 1, "foreignKeys": 2, "orphans": 0},
|
||||||
|
"operation": "schema_check",
|
||||||
|
"orphans": [],
|
||||||
|
"reviewedCandidates": "sha256:" + "b" * 64,
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "succeeded",
|
||||||
|
"workspaceId": "demo",
|
||||||
|
"workspaceRevision": "a" * 40,
|
||||||
|
"zeroOrphans": True,
|
||||||
|
}
|
||||||
|
assert runtime_annotations.read_text() == "tables: {}\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_check_json_reports_orphans_without_prose(tmp_path):
|
||||||
|
cfg = _write_workspace(tmp_path)
|
||||||
|
reviewed = tmp_path / "reviewed.yaml"
|
||||||
|
Annotations(
|
||||||
|
tables={
|
||||||
|
"fact_ablazione": TableAnnotation(
|
||||||
|
foreign_keys=[
|
||||||
|
ForeignKey(
|
||||||
|
columns=["cod_paz"],
|
||||||
|
ref_table="dim_missing",
|
||||||
|
ref_columns=["cod_paz"],
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
).to_yaml(reviewed)
|
||||||
|
|
||||||
|
response = RUNNER.invoke(
|
||||||
|
app,
|
||||||
|
["schema", "check", "-c", str(cfg), "--annotations", str(reviewed), "--json"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.exit_code == 3
|
||||||
|
assert response.stderr == ""
|
||||||
|
payload = json.loads(response.stdout)
|
||||||
|
assert payload["schemaVersion"] == 1
|
||||||
|
assert payload["status"] == "blocked"
|
||||||
|
assert payload["code"] == "annotation_invalid"
|
||||||
|
assert payload["zeroOrphans"] is False
|
||||||
|
assert payload["counts"]["orphans"] == 1
|
||||||
|
assert payload["orphans"] == ["fact_ablazione.fk(cod_paz)->dim_missing"]
|
||||||
|
|
||||||
|
|
||||||
def test_suggest_fks_write_merges_and_is_idempotent(tmp_path):
|
def test_suggest_fks_write_merges_and_is_idempotent(tmp_path):
|
||||||
cfg = _write_workspace(tmp_path)
|
cfg = _write_workspace(tmp_path)
|
||||||
ann_path = tmp_path / "artifacts" / "mschema" / "annotations.yaml"
|
ann_path = tmp_path / "artifacts" / "mschema" / "annotations.yaml"
|
||||||
@@ -211,13 +458,13 @@ def test_suggest_fks_write_merges_and_is_idempotent(tmp_path):
|
|||||||
tables={"fact_ablazione": TableAnnotation(description="Ablazioni")}
|
tables={"fact_ablazione": TableAnnotation(description="Ablazioni")}
|
||||||
).to_yaml(ann_path)
|
).to_yaml(ann_path)
|
||||||
|
|
||||||
res = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg), "--write"])
|
res = RUNNER.invoke(app, ["schema", "suggest-fks", "-c", str(cfg), "--write"])
|
||||||
assert res.exit_code == 0, res.output
|
assert res.exit_code == 0, res.output
|
||||||
ann = Annotations.from_yaml(ann_path)
|
ann = Annotations.from_yaml(ann_path)
|
||||||
assert ann.tables["fact_ablazione"].description == "Ablazioni" # non distrutta
|
assert ann.tables["fact_ablazione"].description == "Ablazioni"
|
||||||
assert len(ann.tables["fact_ablazione"].foreign_keys) == 2
|
assert len(ann.tables["fact_ablazione"].foreign_keys) == 2
|
||||||
|
|
||||||
res2 = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg), "--write"])
|
res2 = RUNNER.invoke(app, ["schema", "suggest-fks", "-c", str(cfg), "--write"])
|
||||||
assert "nessuna FK da suggerire" in res2.output
|
assert "nessuna FK da suggerire" in res2.output
|
||||||
ann2 = Annotations.from_yaml(ann_path)
|
ann2 = Annotations.from_yaml(ann_path)
|
||||||
assert len(ann2.tables["fact_ablazione"].foreign_keys) == 2
|
assert len(ann2.tables["fact_ablazione"].foreign_keys) == 2
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ def build_vector_store(cfg: Config, *, require_write: bool = False) -> VectorSto
|
|||||||
workspace_id=cfg._workspace_id,
|
workspace_id=cfg._workspace_id,
|
||||||
workspace_revision=cfg._workspace_revision,
|
workspace_revision=cfg._workspace_revision,
|
||||||
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
|
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
|
||||||
|
collection_lifecycle=resource.collection_lifecycle,
|
||||||
)
|
)
|
||||||
case other: # pragma: no cover - Pydantic's discriminator rejects this first.
|
case other: # pragma: no cover - Pydantic's discriminator rejects this first.
|
||||||
raise ConfigError(f"Adapter vector non supportato: {other}")
|
raise ConfigError(f"Adapter vector non supportato: {other}")
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ class QdrantVectorStore:
|
|||||||
workspace_id: str,
|
workspace_id: str,
|
||||||
workspace_revision: str | None = None,
|
workspace_revision: str | None = None,
|
||||||
expected_dimension: int | None = None,
|
expected_dimension: int | None = None,
|
||||||
|
collection_lifecycle: str = "self_heal",
|
||||||
request: Callable[..., object] | None = None,
|
request: Callable[..., object] | None = None,
|
||||||
connect_timeout: float = 2.0,
|
connect_timeout: float = 2.0,
|
||||||
read_timeout: float = 10.0,
|
read_timeout: float = 10.0,
|
||||||
@@ -64,6 +65,7 @@ class QdrantVectorStore:
|
|||||||
self._workspace_id = workspace_id
|
self._workspace_id = workspace_id
|
||||||
self._workspace_revision = workspace_revision
|
self._workspace_revision = workspace_revision
|
||||||
self._expected_dimension = expected_dimension
|
self._expected_dimension = expected_dimension
|
||||||
|
self._collection_lifecycle = collection_lifecycle
|
||||||
self._request = request or requests.request
|
self._request = request or requests.request
|
||||||
self._timeout = (connect_timeout, read_timeout)
|
self._timeout = (connect_timeout, read_timeout)
|
||||||
|
|
||||||
@@ -306,6 +308,8 @@ class QdrantVectorStore:
|
|||||||
if response is None:
|
if response is None:
|
||||||
if not strict:
|
if not strict:
|
||||||
raise VectorStoreError("Qdrant collection is missing")
|
raise VectorStoreError("Qdrant collection is missing")
|
||||||
|
if self._collection_lifecycle == "require_existing":
|
||||||
|
raise VectorStoreError("semantic_index_incompatible")
|
||||||
self._call(
|
self._call(
|
||||||
"PUT",
|
"PUT",
|
||||||
f"/collections/{self._collection}",
|
f"/collections/{self._collection}",
|
||||||
@@ -328,9 +332,13 @@ class QdrantVectorStore:
|
|||||||
self._expected_dimension is not None
|
self._expected_dimension is not None
|
||||||
and (size != self._expected_dimension or distance != "Cosine")
|
and (size != self._expected_dimension or distance != "Cosine")
|
||||||
):
|
):
|
||||||
|
if strict and self._collection_lifecycle == "require_existing":
|
||||||
|
raise VectorStoreError("semantic_index_incompatible")
|
||||||
raise VectorStoreError("Qdrant collection configuration mismatch")
|
raise VectorStoreError("Qdrant collection configuration mismatch")
|
||||||
for field_name in _KEYWORD_INDEXES:
|
for field_name in _KEYWORD_INDEXES:
|
||||||
if field_name not in result.get("payload_schema", {}):
|
if field_name not in result.get("payload_schema", {}):
|
||||||
|
if strict and self._collection_lifecycle == "require_existing":
|
||||||
|
raise VectorStoreError("semantic_index_incompatible")
|
||||||
if not strict:
|
if not strict:
|
||||||
raise VectorStoreError("Qdrant collection payload indexes mismatch")
|
raise VectorStoreError("Qdrant collection payload indexes mismatch")
|
||||||
self._call(
|
self._call(
|
||||||
|
|||||||
@@ -2,27 +2,57 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import hashlib
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from tht.cli.config_cmd import CONFIG_OPT
|
from tht.cli.config_cmd import CONFIG_OPT
|
||||||
from tht.config import workspace_id_from_path
|
|
||||||
|
|
||||||
|
|
||||||
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
|
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
|
||||||
|
|
||||||
|
|
||||||
|
def _evidence_json_context(config: Path):
|
||||||
|
from tht.cli.schema_cmd import _load_config_or_exit
|
||||||
|
|
||||||
|
return _load_config_or_exit(config)
|
||||||
|
|
||||||
|
|
||||||
|
def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict:
|
||||||
|
value = {
|
||||||
|
**payload,
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": payload.get("status", "failed"),
|
||||||
|
"code": code,
|
||||||
|
"operation": "preprocess_evidence",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
}
|
||||||
|
if error is not None:
|
||||||
|
value["error"] = error
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _simple_json_payload(*, code: str, error: str) -> dict:
|
||||||
|
return {
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"code": code,
|
||||||
|
"operation": "preprocess_evidence",
|
||||||
|
"error": error,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def run_dwh_from_config(
|
def run_dwh_from_config(
|
||||||
config: Path, *, steps: tuple[str, ...], resume: str | None = None,
|
config: Path, *, steps: tuple[str, ...], resume: str | None = None,
|
||||||
):
|
):
|
||||||
from tht.cli.lsh_cmd import build_lsh_artifacts
|
from tht.cli.lsh_cmd import build_lsh_artifacts
|
||||||
from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog
|
from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog
|
||||||
from tht.jobs.dwh_pipeline import (
|
from tht.jobs.dwh_pipeline import (
|
||||||
DwhPreprocessPipeline, config_dwh_binding,
|
DwhPreprocessPipeline,
|
||||||
|
config_dwh_binding,
|
||||||
)
|
)
|
||||||
|
|
||||||
cfg = _load_config_or_exit(config)
|
cfg = _load_config_or_exit(config)
|
||||||
@@ -87,7 +117,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
|||||||
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
|
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
|
||||||
|
|
||||||
return pipeline.run_as_job(
|
return pipeline.run_as_job(
|
||||||
workspace_id=workspace_id_from_path(config),
|
workspace_id=cfg._workspace_id,
|
||||||
workspace_root=corpus_root.parent,
|
workspace_root=corpus_root.parent,
|
||||||
config_fingerprint=fingerprint(cfg.model_dump_json()),
|
config_fingerprint=fingerprint(cfg.model_dump_json()),
|
||||||
input_fingerprint=fingerprint(config.resolve().as_posix()),
|
input_fingerprint=fingerprint(config.resolve().as_posix()),
|
||||||
@@ -116,7 +146,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
|
|||||||
pipeline_version="evidence-v1",
|
pipeline_version="evidence-v1",
|
||||||
retain_published_generations=cfg.vector.retain_published_generations,
|
retain_published_generations=cfg.vector.retain_published_generations,
|
||||||
)
|
)
|
||||||
pipeline.workspace_id = workspace_id_from_path(config)
|
pipeline.workspace_id = cfg._workspace_id
|
||||||
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
|
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
|
||||||
|
|
||||||
|
|
||||||
@@ -133,7 +163,7 @@ def evidence_cmd(
|
|||||||
if action == "gc":
|
if action == "gc":
|
||||||
try:
|
try:
|
||||||
payload = gc_from_config(config, dry_run=dry_run)
|
payload = gc_from_config(config, dry_run=dry_run)
|
||||||
except Exception:
|
except Exception: # noqa: BLE001
|
||||||
payload = {"status": "failed", "error": "evidence cleanup failed"}
|
payload = {"status": "failed", "error": "evidence cleanup failed"}
|
||||||
if json_output:
|
if json_output:
|
||||||
typer.echo(json.dumps(payload, sort_keys=True))
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
@@ -145,8 +175,12 @@ def evidence_cmd(
|
|||||||
else:
|
else:
|
||||||
typer.echo(f"OK: evicted={len(payload['evicted'])} failures={len(payload['failures'])}")
|
typer.echo(f"OK: evicted={len(payload['evicted'])} failures={len(payload['failures'])}")
|
||||||
return
|
return
|
||||||
|
cfg = _evidence_json_context(config) if json_output else None
|
||||||
if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None:
|
if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None:
|
||||||
payload = {"status": "failed", "error": "resume requires a preprocessing run id"}
|
payload = _simple_json_payload(
|
||||||
|
code="invalid_resume",
|
||||||
|
error="resume requires a preprocessing run id",
|
||||||
|
)
|
||||||
if json_output:
|
if json_output:
|
||||||
typer.echo(json.dumps(payload, sort_keys=True))
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
else:
|
else:
|
||||||
@@ -154,23 +188,43 @@ def evidence_cmd(
|
|||||||
raise typer.Exit(code=2)
|
raise typer.Exit(code=2)
|
||||||
try:
|
try:
|
||||||
result = run_from_config(config, dry_run=dry_run, resume=resume)
|
result = run_from_config(config, dry_run=dry_run, resume=resume)
|
||||||
except Exception:
|
except Exception: # noqa: BLE001
|
||||||
payload = {"status": "failed", "error": "preprocessing failed"}
|
payload = {"status": "failed"}
|
||||||
if json_output:
|
if json_output:
|
||||||
typer.echo(json.dumps(payload, sort_keys=True))
|
typer.echo(json.dumps(
|
||||||
|
_evidence_json_payload(
|
||||||
|
cfg,
|
||||||
|
payload,
|
||||||
|
code="preprocessing_failed",
|
||||||
|
error="preprocessing failed",
|
||||||
|
),
|
||||||
|
sort_keys=True,
|
||||||
|
))
|
||||||
else:
|
else:
|
||||||
typer.secho("ERRORE: preprocessing failed", fg=typer.colors.RED, err=True)
|
typer.secho("ERRORE: preprocessing failed", fg=typer.colors.RED, err=True)
|
||||||
raise typer.Exit(code=1) from None
|
raise typer.Exit(code=1) from None
|
||||||
payload = result.model_dump(mode="json")
|
payload = result.model_dump(mode="json")
|
||||||
if payload.get("status") != "succeeded":
|
if payload.get("status") != "succeeded":
|
||||||
payload["error"] = "preprocessing job failed"
|
|
||||||
if json_output:
|
if json_output:
|
||||||
typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
typer.echo(json.dumps(
|
||||||
|
_evidence_json_payload(
|
||||||
|
cfg,
|
||||||
|
payload,
|
||||||
|
code="preprocessing_failed",
|
||||||
|
error="preprocessing job failed",
|
||||||
|
),
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
))
|
||||||
else:
|
else:
|
||||||
typer.secho("ERRORE: preprocessing job failed", fg=typer.colors.RED, err=True)
|
typer.secho("ERRORE: preprocessing job failed", fg=typer.colors.RED, err=True)
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
if json_output:
|
if json_output:
|
||||||
typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
typer.echo(json.dumps(
|
||||||
|
_evidence_json_payload(cfg, payload, code="ok"),
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
))
|
||||||
else:
|
else:
|
||||||
counts = payload["counts"]
|
counts = payload["counts"]
|
||||||
typer.echo(
|
typer.echo(
|
||||||
@@ -205,7 +259,7 @@ def dwh_cmd(
|
|||||||
raise typer.Exit(code=2)
|
raise typer.Exit(code=2)
|
||||||
try:
|
try:
|
||||||
result = run_dwh_from_config(config, steps=selected, resume=resume)
|
result = run_dwh_from_config(config, steps=selected, resume=resume)
|
||||||
except Exception:
|
except Exception: # noqa: BLE001
|
||||||
payload = {"status": "failed", "error": "DWH preprocessing failed"}
|
payload = {"status": "failed", "error": "DWH preprocessing failed"}
|
||||||
if json_output:
|
if json_output:
|
||||||
typer.echo(json.dumps(payload, sort_keys=True))
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
|
|||||||
+344
-130
@@ -1,7 +1,11 @@
|
|||||||
from pathlib import Path
|
import hashlib
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
import yaml
|
||||||
|
|
||||||
from tht.adapters.factory import build_dwh
|
from tht.adapters.factory import build_dwh
|
||||||
from tht.cli.config_cmd import CONFIG_OPT
|
from tht.cli.config_cmd import CONFIG_OPT
|
||||||
from tht.config import ConfigError, load_config
|
from tht.config import ConfigError, load_config
|
||||||
@@ -21,7 +25,7 @@ def _add_examples(dwh, phys, examples) -> None:
|
|||||||
sampled = dwh.sample_column(
|
sampled = dwh.sample_column(
|
||||||
table_name, column_name, limit=examples.max_per_column
|
table_name, column_name, limit=examples.max_per_column
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc: # noqa: BLE001
|
||||||
logger.warning("Campionamento saltato per %s.%s: %s",
|
logger.warning("Campionamento saltato per %s.%s: %s",
|
||||||
table_name, column_name, exc)
|
table_name, column_name, exc)
|
||||||
continue
|
continue
|
||||||
@@ -83,7 +87,7 @@ def introspect_cmd(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
cached = PhysicalSchema.from_yaml(out)
|
cached = PhysicalSchema.from_yaml(out)
|
||||||
except Exception:
|
except Exception: # noqa: BLE001,S110
|
||||||
pass # catalogo illeggibile: procedi con la re-introspezione
|
pass # catalogo illeggibile: procedi con la re-introspezione
|
||||||
else:
|
else:
|
||||||
ts = cached.introspected_at
|
ts = cached.introspected_at
|
||||||
@@ -105,7 +109,7 @@ def introspect_cmd(
|
|||||||
raise RuntimeError("DWH preprocessing failed")
|
raise RuntimeError("DWH preprocessing failed")
|
||||||
out = physical_path(cfg)
|
out = physical_path(cfg)
|
||||||
phys = PhysicalSchema.from_yaml(out)
|
phys = PhysicalSchema.from_yaml(out)
|
||||||
except Exception as e:
|
except Exception as e: # noqa: BLE001
|
||||||
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
|
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
n_cols = sum(len(t.columns) for t in phys.tables.values())
|
n_cols = sum(len(t.columns) for t in phys.tables.values())
|
||||||
@@ -119,8 +123,188 @@ def introspect_cmd(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _json_sha(value) -> str:
|
||||||
|
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
|
return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _emit_json(payload: dict) -> None:
|
||||||
|
typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
def _sorted_fk_payloads(foreign_keys) -> list[dict]:
|
||||||
|
payloads = [fk.model_dump(mode="json", exclude_defaults=True) for fk in foreign_keys]
|
||||||
|
return sorted(
|
||||||
|
payloads,
|
||||||
|
key=lambda payload: (
|
||||||
|
tuple(payload.get("columns", [])),
|
||||||
|
payload.get("ref_table", ""),
|
||||||
|
tuple(payload.get("ref_columns", [])),
|
||||||
|
payload.get("name", ""),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sorted_annotations_payload(annotations) -> dict:
|
||||||
|
tables = {}
|
||||||
|
for table_name in sorted(annotations.tables):
|
||||||
|
table = annotations.tables[table_name]
|
||||||
|
payload = {}
|
||||||
|
if table.description:
|
||||||
|
payload["description"] = table.description
|
||||||
|
if table.concepts:
|
||||||
|
payload["concepts"] = table.concepts
|
||||||
|
if table.notes:
|
||||||
|
payload["notes"] = table.notes
|
||||||
|
if table.columns:
|
||||||
|
payload["columns"] = {
|
||||||
|
name: value.model_dump(mode="json", exclude_defaults=True)
|
||||||
|
for name, value in sorted(table.columns.items())
|
||||||
|
}
|
||||||
|
if table.foreign_keys:
|
||||||
|
payload["foreign_keys"] = _sorted_fk_payloads(table.foreign_keys)
|
||||||
|
tables[table_name] = payload
|
||||||
|
return {"tables": tables}
|
||||||
|
|
||||||
|
|
||||||
|
def _suggested_fk_payload(annotations_by_table: dict) -> dict:
|
||||||
|
return {
|
||||||
|
"tables": {
|
||||||
|
table_name: {"foreign_keys": _sorted_fk_payloads(foreign_keys)}
|
||||||
|
for table_name, foreign_keys in sorted(annotations_by_table.items())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _load_sql_inputs(entries: list[Path] | None) -> list[tuple[str, str]]:
|
||||||
|
max_file_bytes = 1024 * 1024
|
||||||
|
max_total_bytes = 16 * 1024 * 1024
|
||||||
|
total_bytes = 0
|
||||||
|
sql_files: list[Path] = []
|
||||||
|
for entry in entries or []:
|
||||||
|
if entry.is_dir():
|
||||||
|
sql_files.extend(sorted(path for path in entry.rglob("*.sql") if path.is_file()))
|
||||||
|
continue
|
||||||
|
sql_files.append(entry)
|
||||||
|
loaded = []
|
||||||
|
for sql_file in sorted(sql_files, key=lambda candidate: candidate.as_posix()):
|
||||||
|
if not sql_file.exists() or not sql_file.is_file() or sql_file.is_symlink():
|
||||||
|
raise ValueError("invalid SQL input")
|
||||||
|
size = sql_file.stat().st_size
|
||||||
|
total_bytes += size
|
||||||
|
if size > max_file_bytes or total_bytes > max_total_bytes:
|
||||||
|
raise ValueError("invalid SQL input")
|
||||||
|
loaded.append((sql_file.as_posix(), sql_file.read_text(encoding="utf-8")))
|
||||||
|
return loaded
|
||||||
|
|
||||||
|
|
||||||
|
def _suggest_fk_result(physical, annotations, *, sql_inputs: list[tuple[str, str]], assume: list[str] | None):
|
||||||
|
from tht.mschema.fkmine import mine_join_pairs
|
||||||
|
from tht.mschema.models import ForeignKey
|
||||||
|
|
||||||
|
assumed: dict[str, str] = {}
|
||||||
|
for value in assume or []:
|
||||||
|
col, _, ref = value.partition("=")
|
||||||
|
if not ref or ref not in physical.tables:
|
||||||
|
raise ValueError("invalid assume mapping")
|
||||||
|
assumed[col] = ref
|
||||||
|
|
||||||
|
def _single_pk(table) -> str | None:
|
||||||
|
pks = [column_name for column_name, column in table.columns.items() if column.pk]
|
||||||
|
return pks[0] if len(pks) == 1 else None
|
||||||
|
|
||||||
|
pk_owners: dict[str, list[str]] = {}
|
||||||
|
for table_name, table in physical.tables.items():
|
||||||
|
pk = _single_pk(table)
|
||||||
|
if pk:
|
||||||
|
pk_owners.setdefault(pk, []).append(table_name)
|
||||||
|
|
||||||
|
dim_time_pk = None
|
||||||
|
if "dim_time" in physical.tables:
|
||||||
|
dim_time_pk = _single_pk(physical.tables["dim_time"])
|
||||||
|
|
||||||
|
def _known(table_name: str) -> set:
|
||||||
|
keys = set()
|
||||||
|
for fk in physical.tables[table_name].foreign_keys:
|
||||||
|
keys.add((tuple(fk.columns), fk.ref_table, tuple(fk.ref_columns)))
|
||||||
|
annotation = annotations.tables.get(table_name)
|
||||||
|
if annotation:
|
||||||
|
for fk in annotation.foreign_keys:
|
||||||
|
keys.add((tuple(fk.columns), fk.ref_table, tuple(fk.ref_columns)))
|
||||||
|
return keys
|
||||||
|
|
||||||
|
known_by_table: dict[str, set] = {table_name: _known(table_name) for table_name in physical.tables}
|
||||||
|
suggested: dict[str, list[ForeignKey]] = {}
|
||||||
|
|
||||||
|
def _add(table_name: str, column_name: str, ref_table: str, ref_column: str) -> None:
|
||||||
|
key = ((column_name,), ref_table, (ref_column,))
|
||||||
|
if key in known_by_table[table_name]:
|
||||||
|
return
|
||||||
|
known_by_table[table_name].add(key)
|
||||||
|
suggested.setdefault(table_name, []).append(
|
||||||
|
ForeignKey(columns=[column_name], ref_table=ref_table, ref_columns=[ref_column])
|
||||||
|
)
|
||||||
|
|
||||||
|
mined_total = 0
|
||||||
|
for _name, sql_text in sql_inputs:
|
||||||
|
pairs = mine_join_pairs(sql_text, physical)
|
||||||
|
mined_total += sum(pairs.values())
|
||||||
|
for src_t, src_c, ref_t, ref_c in pairs:
|
||||||
|
_add(src_t, src_c, ref_t, ref_c)
|
||||||
|
|
||||||
|
ambiguous_skipped: set[str] = set()
|
||||||
|
for table_name, table in physical.tables.items():
|
||||||
|
for column_name in table.columns:
|
||||||
|
if dim_time_pk and column_name.endswith("time_key") and table_name != "dim_time":
|
||||||
|
_add(table_name, column_name, "dim_time", dim_time_pk)
|
||||||
|
continue
|
||||||
|
if column_name in assumed:
|
||||||
|
if assumed[column_name] != table_name:
|
||||||
|
_add(table_name, column_name, assumed[column_name], column_name)
|
||||||
|
continue
|
||||||
|
owners = [owner for owner in pk_owners.get(column_name, []) if owner != table_name]
|
||||||
|
if not owners or column_name in _GENERIC_PK_NAMES:
|
||||||
|
continue
|
||||||
|
if len(pk_owners[column_name]) > 1:
|
||||||
|
ambiguous_skipped.add(column_name)
|
||||||
|
continue
|
||||||
|
_add(table_name, column_name, owners[0], column_name)
|
||||||
|
|
||||||
|
candidate_annotations = _suggested_fk_payload(suggested)
|
||||||
|
candidate_yaml = yaml.safe_dump(candidate_annotations, sort_keys=False, allow_unicode=True)
|
||||||
|
counts = {
|
||||||
|
"ambiguousColumns": len(ambiguous_skipped),
|
||||||
|
"candidateTables": len(candidate_annotations["tables"]),
|
||||||
|
"candidates": sum(len(value["foreign_keys"]) for value in candidate_annotations["tables"].values()),
|
||||||
|
"minedJoins": mined_total,
|
||||||
|
"sqlFiles": len(sql_inputs),
|
||||||
|
}
|
||||||
|
candidate_document = {
|
||||||
|
"annotations": candidate_annotations,
|
||||||
|
"counts": {
|
||||||
|
"candidateTables": counts["candidateTables"],
|
||||||
|
"candidates": counts["candidates"],
|
||||||
|
},
|
||||||
|
"schemaVersion": 1,
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"ambiguous": sorted(ambiguous_skipped),
|
||||||
|
"candidate_count": counts["candidates"],
|
||||||
|
"candidateDigest": "sha256:" + hashlib.sha256(candidate_yaml.encode("utf-8")).hexdigest(),
|
||||||
|
"candidateDocument": candidate_document,
|
||||||
|
"candidate_yaml": candidate_yaml,
|
||||||
|
"counts": counts,
|
||||||
|
"suggested": suggested,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@schema_app.command("check")
|
@schema_app.command("check")
|
||||||
def check_cmd(config: Path = CONFIG_OPT) -> None:
|
def check_cmd(
|
||||||
|
config: Path = CONFIG_OPT,
|
||||||
|
annotations: Path | None = typer.Option(None, "--annotations"), # noqa: B008
|
||||||
|
reviewed_candidates: str | None = typer.Option(None, "--reviewed-candidates"),
|
||||||
|
json_output: bool = typer.Option(False, "--json"),
|
||||||
|
) -> None:
|
||||||
"""Confronta physical.yaml e annotations.yaml; segnala annotazioni orfane."""
|
"""Confronta physical.yaml e annotations.yaml; segnala annotazioni orfane."""
|
||||||
from tht.mschema.merge import find_orphans
|
from tht.mschema.merge import find_orphans
|
||||||
from tht.mschema.models import Annotations, PhysicalSchema
|
from tht.mschema.models import Annotations, PhysicalSchema
|
||||||
@@ -128,20 +312,80 @@ def check_cmd(config: Path = CONFIG_OPT) -> None:
|
|||||||
cfg = _load_config_or_exit(config)
|
cfg = _load_config_or_exit(config)
|
||||||
phys_file = physical_path(cfg)
|
phys_file = physical_path(cfg)
|
||||||
if not phys_file.exists():
|
if not phys_file.exists():
|
||||||
typer.secho(
|
if json_output:
|
||||||
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
_emit_json({
|
||||||
fg=typer.colors.RED, err=True,
|
"code": "schema_missing",
|
||||||
)
|
"error": "physical schema is missing",
|
||||||
|
"operation": "schema_check",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
typer.secho(
|
||||||
|
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
||||||
|
fg=typer.colors.RED,
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
physical = PhysicalSchema.from_yaml(phys_file)
|
physical = PhysicalSchema.from_yaml(phys_file)
|
||||||
annotations = Annotations.from_yaml(annotations_path(cfg))
|
annotations_file = annotations or annotations_path(cfg)
|
||||||
|
try:
|
||||||
|
loaded_annotations = Annotations.from_yaml(annotations_file)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
if json_output:
|
||||||
|
_emit_json({
|
||||||
|
"code": "annotation_invalid",
|
||||||
|
"error": "annotations are invalid",
|
||||||
|
"operation": "schema_check",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
typer.secho("ERRORE: annotations non valide.", fg=typer.colors.RED, err=True)
|
||||||
|
raise typer.Exit(code=1) from None
|
||||||
|
|
||||||
ignored = [
|
ignored = [
|
||||||
f"{t}.{c} ({col.eligibility_reason})"
|
f"{table_name}.{column_name} ({column.eligibility_reason})"
|
||||||
for t, table in physical.tables.items()
|
for table_name, table in physical.tables.items()
|
||||||
for c, col in table.columns.items()
|
for column_name, column in table.columns.items()
|
||||||
if not col.eligible
|
if not column.eligible
|
||||||
]
|
]
|
||||||
|
orphans = sorted(find_orphans(physical, loaded_annotations))
|
||||||
|
if json_output:
|
||||||
|
annotations_payload = _sorted_annotations_payload(loaded_annotations)
|
||||||
|
payload = {
|
||||||
|
"annotationsDigest": _json_sha({"annotations": annotations_payload, "schemaVersion": 1}),
|
||||||
|
"code": "ok" if not orphans else "annotation_invalid",
|
||||||
|
"counts": {
|
||||||
|
"annotationTables": len(annotations_payload["tables"]),
|
||||||
|
"foreignKeys": sum(
|
||||||
|
len(table_payload.get("foreign_keys", []))
|
||||||
|
for table_payload in annotations_payload["tables"].values()
|
||||||
|
),
|
||||||
|
"orphans": len(orphans),
|
||||||
|
},
|
||||||
|
"operation": "schema_check",
|
||||||
|
"orphan_count": len(orphans),
|
||||||
|
"orphans": orphans,
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "succeeded" if not orphans else "blocked",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
"zeroOrphans": not orphans,
|
||||||
|
}
|
||||||
|
payload["annotations_digest"] = "sha256:" + hashlib.sha256(Path(annotations_file).read_bytes()).hexdigest()
|
||||||
|
if reviewed_candidates is not None:
|
||||||
|
payload["reviewedCandidates"] = reviewed_candidates
|
||||||
|
payload["reviewed_candidates_digest"] = reviewed_candidates
|
||||||
|
_emit_json(payload)
|
||||||
|
if orphans:
|
||||||
|
raise typer.Exit(code=3)
|
||||||
|
return
|
||||||
|
|
||||||
if ignored:
|
if ignored:
|
||||||
typer.secho(
|
typer.secho(
|
||||||
f"Colonne ignorate (testo ampio, {len(ignored)}):", fg=typer.colors.YELLOW
|
f"Colonne ignorate (testo ampio, {len(ignored)}):", fg=typer.colors.YELLOW
|
||||||
@@ -149,11 +393,10 @@ def check_cmd(config: Path = CONFIG_OPT) -> None:
|
|||||||
for line in ignored:
|
for line in ignored:
|
||||||
typer.echo(f" - {line}")
|
typer.echo(f" - {line}")
|
||||||
|
|
||||||
orphans = find_orphans(physical, annotations)
|
|
||||||
if orphans:
|
if orphans:
|
||||||
typer.secho(f"ATTENZIONE: {len(orphans)} annotazioni orfane:", fg=typer.colors.YELLOW)
|
typer.secho(f"ATTENZIONE: {len(orphans)} annotazioni orfane:", fg=typer.colors.YELLOW)
|
||||||
for o in orphans:
|
for orphan in orphans:
|
||||||
typer.echo(f" - {o}")
|
typer.echo(f" - {orphan}")
|
||||||
raise typer.Exit(code=3)
|
raise typer.Exit(code=3)
|
||||||
typer.secho("OK: nessuna annotazione orfana.", fg=typer.colors.GREEN)
|
typer.secho("OK: nessuna annotazione orfana.", fg=typer.colors.GREEN)
|
||||||
|
|
||||||
@@ -166,11 +409,11 @@ _GENERIC_PK_NAMES = {"id", "key", "code"}
|
|||||||
@schema_app.command("suggest-fks")
|
@schema_app.command("suggest-fks")
|
||||||
def suggest_fks_cmd(
|
def suggest_fks_cmd(
|
||||||
config: Path = CONFIG_OPT,
|
config: Path = CONFIG_OPT,
|
||||||
from_sql: list[Path] = typer.Option(
|
from_sql: list[Path] = typer.Option( # noqa: B008
|
||||||
None, "--from-sql",
|
None, "--from-sql",
|
||||||
help="Directory di .sql approvati da cui minare i join reali (ripetibile).",
|
help="Directory o file .sql approvati da cui minare i join reali (ripetibile).",
|
||||||
),
|
),
|
||||||
assume: list[str] = typer.Option(
|
assume: list[str] = typer.Option( # noqa: B008
|
||||||
None, "--assume",
|
None, "--assume",
|
||||||
help="Disambigua una PK con piu' proprietari: col=tabella_ref "
|
help="Disambigua una PK con piu' proprietari: col=tabella_ref "
|
||||||
"(es. cod_paz=dim_patient). Ripetibile.",
|
"(es. cod_paz=dim_patient). Ripetibile.",
|
||||||
@@ -179,132 +422,103 @@ def suggest_fks_cmd(
|
|||||||
False, "--write",
|
False, "--write",
|
||||||
help="Fonde i suggerimenti in annotations.yaml (aggiunge solo FK mancanti).",
|
help="Fonde i suggerimenti in annotations.yaml (aggiunge solo FK mancanti).",
|
||||||
),
|
),
|
||||||
|
json_output: bool = typer.Option(False, "--json"),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Suggerisce FK logiche per la curazione umana in annotations.yaml.
|
"""Suggerisce FK logiche per la curazione umana in annotations.yaml."""
|
||||||
|
|
||||||
Tre regole, in ordine di confidenza: (1) equi-join minati dall'SQL gia'
|
|
||||||
approvato (--from-sql); (2) colonna `*time_key` verso la PK di dim_time;
|
|
||||||
(3) colonna con lo stesso nome della PK di UN'ALTRA tabella, solo se quel
|
|
||||||
nome ha un unico proprietario e non e' generico (id/key/code) — salvo
|
|
||||||
disambiguazione esplicita con --assume.
|
|
||||||
"""
|
|
||||||
import yaml as _yaml
|
import yaml as _yaml
|
||||||
|
|
||||||
from tht.mschema.fkmine import mine_join_pairs
|
from tht.mschema.models import Annotations, PhysicalSchema, TableAnnotation
|
||||||
from tht.mschema.models import Annotations, ForeignKey, PhysicalSchema, TableAnnotation
|
|
||||||
|
|
||||||
cfg = _load_config_or_exit(config)
|
cfg = _load_config_or_exit(config)
|
||||||
phys_file = physical_path(cfg)
|
phys_file = physical_path(cfg)
|
||||||
if not phys_file.exists():
|
if not phys_file.exists():
|
||||||
typer.secho(
|
if json_output:
|
||||||
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
_emit_json({
|
||||||
fg=typer.colors.RED, err=True,
|
"code": "schema_missing",
|
||||||
)
|
"error": "physical schema is missing",
|
||||||
|
"operation": "schema_suggest_fks",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
typer.secho(
|
||||||
|
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
||||||
|
fg=typer.colors.RED,
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
physical = PhysicalSchema.from_yaml(phys_file)
|
physical = PhysicalSchema.from_yaml(phys_file)
|
||||||
ann_path = annotations_path(cfg)
|
ann_path = annotations_path(cfg)
|
||||||
annotations = Annotations.from_yaml(ann_path)
|
loaded_annotations = Annotations.from_yaml(ann_path)
|
||||||
|
try:
|
||||||
assumed: dict[str, str] = {}
|
sql_inputs = _load_sql_inputs(from_sql)
|
||||||
for a in assume or []:
|
result = _suggest_fk_result(physical, loaded_annotations, sql_inputs=sql_inputs, assume=assume)
|
||||||
col, _, ref = a.partition("=")
|
except ValueError as exc:
|
||||||
if not ref or ref not in physical.tables:
|
code = "invalid_argument"
|
||||||
typer.secho(
|
error = str(exc)
|
||||||
f"ERRORE: --assume '{a}' non valido (atteso col=tabella nel catalogo).",
|
if json_output:
|
||||||
fg=typer.colors.RED, err=True,
|
_emit_json({
|
||||||
|
"code": code,
|
||||||
|
"error": error,
|
||||||
|
"operation": "schema_suggest_fks",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
human_error = (
|
||||||
|
"--assume non valido (atteso col=tabella nel catalogo)."
|
||||||
|
if error == "invalid assume mapping"
|
||||||
|
else error
|
||||||
)
|
)
|
||||||
raise typer.Exit(code=1)
|
typer.secho(f"ERRORE: {human_error}", fg=typer.colors.RED, err=True)
|
||||||
assumed[col] = ref
|
raise typer.Exit(code=1) from None
|
||||||
|
|
||||||
def _single_pk(table) -> str | None:
|
if json_output:
|
||||||
pks = [c for c, col in table.columns.items() if col.pk]
|
_emit_json({
|
||||||
return pks[0] if len(pks) == 1 else None
|
"candidate_count": result["candidate_count"],
|
||||||
|
"candidateDigest": result["candidateDigest"],
|
||||||
|
"candidateDocument": result["candidateDocument"],
|
||||||
|
"candidate_yaml": result["candidate_yaml"],
|
||||||
|
"code": "ok",
|
||||||
|
"counts": result["counts"],
|
||||||
|
"operation": "schema_suggest_fks",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "succeeded",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
|
||||||
pk_owners: dict[str, list[str]] = {}
|
if result["counts"]["sqlFiles"]:
|
||||||
for tname, table in physical.tables.items():
|
|
||||||
pk = _single_pk(table)
|
|
||||||
if pk:
|
|
||||||
pk_owners.setdefault(pk, []).append(tname)
|
|
||||||
|
|
||||||
dim_time_pk = None
|
|
||||||
if "dim_time" in physical.tables:
|
|
||||||
dim_time_pk = _single_pk(physical.tables["dim_time"])
|
|
||||||
|
|
||||||
def _known(tname: str) -> set:
|
|
||||||
keys = set()
|
|
||||||
for fk in physical.tables[tname].foreign_keys:
|
|
||||||
keys.add((tuple(fk.columns), fk.ref_table, tuple(fk.ref_columns)))
|
|
||||||
ann = annotations.tables.get(tname)
|
|
||||||
if ann:
|
|
||||||
for fk in ann.foreign_keys:
|
|
||||||
keys.add((tuple(fk.columns), fk.ref_table, tuple(fk.ref_columns)))
|
|
||||||
return keys
|
|
||||||
|
|
||||||
known_by_table: dict[str, set] = {t: _known(t) for t in physical.tables}
|
|
||||||
suggested: dict[str, list[ForeignKey]] = {}
|
|
||||||
|
|
||||||
def _add(tname: str, col: str, ref_table: str, ref_col: str) -> None:
|
|
||||||
key = ((col,), ref_table, (ref_col,))
|
|
||||||
if key in known_by_table[tname]:
|
|
||||||
return
|
|
||||||
known_by_table[tname].add(key)
|
|
||||||
suggested.setdefault(tname, []).append(
|
|
||||||
ForeignKey(columns=[col], ref_table=ref_table, ref_columns=[ref_col])
|
|
||||||
)
|
|
||||||
|
|
||||||
# Regola 1: join minati dall'SQL approvato.
|
|
||||||
n_sql_files = 0
|
|
||||||
mined_total = 0
|
|
||||||
for d in from_sql or []:
|
|
||||||
for sql_file in sorted(d.rglob("*.sql")):
|
|
||||||
n_sql_files += 1
|
|
||||||
pairs = mine_join_pairs(sql_file.read_text(), physical)
|
|
||||||
mined_total += sum(pairs.values())
|
|
||||||
for (src_t, src_c, ref_t, ref_c) in pairs:
|
|
||||||
_add(src_t, src_c, ref_t, ref_c)
|
|
||||||
|
|
||||||
# Regole 2 e 3: convenzioni di naming.
|
|
||||||
ambiguous_skipped: set[str] = set()
|
|
||||||
for tname, table in physical.tables.items():
|
|
||||||
for cname in table.columns:
|
|
||||||
if dim_time_pk and cname.endswith("time_key") and tname != "dim_time":
|
|
||||||
_add(tname, cname, "dim_time", dim_time_pk)
|
|
||||||
continue
|
|
||||||
if cname in assumed:
|
|
||||||
if assumed[cname] != tname:
|
|
||||||
_add(tname, cname, assumed[cname], cname)
|
|
||||||
continue
|
|
||||||
owners = [o for o in pk_owners.get(cname, []) if o != tname]
|
|
||||||
if not owners or cname in _GENERIC_PK_NAMES:
|
|
||||||
continue
|
|
||||||
if len(pk_owners[cname]) > 1:
|
|
||||||
ambiguous_skipped.add(cname)
|
|
||||||
continue
|
|
||||||
_add(tname, cname, owners[0], cname)
|
|
||||||
|
|
||||||
if n_sql_files:
|
|
||||||
typer.secho(
|
typer.secho(
|
||||||
f"Minati {mined_total} equi-join da {n_sql_files} file SQL.",
|
f"Minati {result['counts']['minedJoins']} equi-join da {result['counts']['sqlFiles']} file SQL.",
|
||||||
fg=typer.colors.BLUE, err=True,
|
fg=typer.colors.BLUE,
|
||||||
|
err=True,
|
||||||
)
|
)
|
||||||
if ambiguous_skipped:
|
if result["ambiguous"]:
|
||||||
typer.secho(
|
typer.secho(
|
||||||
"PK ambigue saltate dalla regola same-name (piu' tabelle proprietarie): "
|
"PK ambigue saltate dalla regola same-name (piu' tabelle proprietarie): "
|
||||||
+ ", ".join(sorted(ambiguous_skipped))
|
+ ", ".join(result["ambiguous"])
|
||||||
+ ". Se servono, aggiungile a mano o passa --from-sql.",
|
+ ". Se servono, aggiungile a mano o passa --from-sql.",
|
||||||
fg=typer.colors.YELLOW, err=True,
|
fg=typer.colors.YELLOW,
|
||||||
|
err=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
n_fks = sum(len(v) for v in suggested.values())
|
suggested = result["suggested"]
|
||||||
|
n_fks = result["counts"]["candidates"]
|
||||||
if not suggested:
|
if not suggested:
|
||||||
typer.secho("OK: nessuna FK da suggerire.", fg=typer.colors.GREEN)
|
typer.secho("OK: nessuna FK da suggerire.", fg=typer.colors.GREEN)
|
||||||
return
|
return
|
||||||
|
|
||||||
if write:
|
if write:
|
||||||
for tname, fks in suggested.items():
|
for table_name, foreign_keys in suggested.items():
|
||||||
ann = annotations.tables.setdefault(tname, TableAnnotation())
|
annotation = loaded_annotations.tables.setdefault(table_name, TableAnnotation())
|
||||||
ann.foreign_keys.extend(fks)
|
annotation.foreign_keys.extend(foreign_keys)
|
||||||
annotations.to_yaml(ann_path)
|
loaded_annotations.to_yaml(ann_path)
|
||||||
typer.secho(
|
typer.secho(
|
||||||
f"OK: {n_fks} FK suggerite aggiunte a {ann_path} "
|
f"OK: {n_fks} FK suggerite aggiunte a {ann_path} "
|
||||||
f"({len(suggested)} tabelle). Rivedile a mano prima dell'uso.",
|
f"({len(suggested)} tabelle). Rivedile a mano prima dell'uso.",
|
||||||
@@ -312,13 +526,13 @@ def suggest_fks_cmd(
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
payload = {
|
typer.echo(
|
||||||
"tables": {
|
_yaml.safe_dump(
|
||||||
tname: {"foreign_keys": [fk.model_dump(exclude_defaults=True) for fk in fks]}
|
result["candidateDocument"]["annotations"],
|
||||||
for tname, fks in suggested.items()
|
sort_keys=False,
|
||||||
}
|
allow_unicode=True,
|
||||||
}
|
)
|
||||||
typer.echo(_yaml.safe_dump(payload, sort_keys=False, allow_unicode=True))
|
)
|
||||||
typer.secho(
|
typer.secho(
|
||||||
f"{n_fks} FK candidate ({len(suggested)} tabelle). "
|
f"{n_fks} FK candidate ({len(suggested)} tabelle). "
|
||||||
f"Usa --write per fonderle in annotations.yaml, poi curale a mano.",
|
f"Usa --write per fonderle in annotations.yaml, poi curale a mano.",
|
||||||
@@ -332,10 +546,10 @@ def render_cmd(
|
|||||||
format: str = typer.Option(
|
format: str = typer.Option(
|
||||||
"markdown", "--format", "-f", help="Formato: markdown | mschema-text | schema-dict"
|
"markdown", "--format", "-f", help="Formato: markdown | mschema-text | schema-dict"
|
||||||
),
|
),
|
||||||
tables: list[str] = typer.Option(
|
tables: list[str] = typer.Option( # noqa: B008
|
||||||
None, "--table", "-t", help="Limita alle tabelle indicate (ripetibile)."
|
None, "--table", "-t", help="Limita alle tabelle indicate (ripetibile)."
|
||||||
),
|
),
|
||||||
output: Path = typer.Option(None, "--output", "-o", help="File di output (default stdout)."),
|
output: Path = typer.Option(None, "--output", "-o", help="File di output (default stdout)."), # noqa: B008
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Serializza mschema (physical + annotations) nel formato richiesto."""
|
"""Serializza mschema (physical + annotations) nel formato richiesto."""
|
||||||
import json
|
import json
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import hashlib
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import typer
|
import typer
|
||||||
@@ -11,6 +13,14 @@ from tht.vectorstore.store import SyncStats, content_hash
|
|||||||
vector_app = typer.Typer(help="Indice semantico Qdrant (derivato, rigenerabile)")
|
vector_app = typer.Typer(help="Indice semantico Qdrant (derivato, rigenerabile)")
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact_digest(path: Path) -> str:
|
||||||
|
return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _emit_json(payload: dict) -> None:
|
||||||
|
typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
def make_embedder(embeddings_cfg):
|
def make_embedder(embeddings_cfg):
|
||||||
"""Factory del client embeddings (monkeypatchabile nei test)."""
|
"""Factory del client embeddings (monkeypatchabile nei test)."""
|
||||||
from tht.vectorstore.embeddings import OllamaEmbeddings
|
from tht.vectorstore.embeddings import OllamaEmbeddings
|
||||||
@@ -117,9 +127,14 @@ def init_cmd(
|
|||||||
|
|
||||||
|
|
||||||
@vector_app.command("index-schema")
|
@vector_app.command("index-schema")
|
||||||
def index_schema_cmd(config: Path = CONFIG_OPT) -> None:
|
def index_schema_cmd(
|
||||||
|
config: Path = CONFIG_OPT,
|
||||||
|
json_output: bool = typer.Option(False, "--json"),
|
||||||
|
) -> None:
|
||||||
"""Embedda e sincronizza i record schema (tabelle e colonne) nel semantic store."""
|
"""Embedda e sincronizza i record schema (tabelle e colonne) nel semantic store."""
|
||||||
|
from tht.adapters.factory import build_vector_store
|
||||||
from tht.mschema.models import Annotations, PhysicalSchema
|
from tht.mschema.models import Annotations, PhysicalSchema
|
||||||
|
from tht.ports.vector import VectorStoreError
|
||||||
from tht.vectorstore.records import schema_records
|
from tht.vectorstore.records import schema_records
|
||||||
|
|
||||||
cfg = _load_config_or_exit(config)
|
cfg = _load_config_or_exit(config)
|
||||||
@@ -127,20 +142,69 @@ def index_schema_cmd(config: Path = CONFIG_OPT) -> None:
|
|||||||
require_vector_cfg(cfg)
|
require_vector_cfg(cfg)
|
||||||
phys_file = physical_path(cfg)
|
phys_file = physical_path(cfg)
|
||||||
if not phys_file.exists():
|
if not phys_file.exists():
|
||||||
typer.secho(
|
message = f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`."
|
||||||
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
if json_output:
|
||||||
fg=typer.colors.RED, err=True,
|
_emit_json({
|
||||||
)
|
"code": "schema_missing",
|
||||||
|
"error": "physical schema is missing",
|
||||||
|
"operation": "index_schema",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
typer.secho(message, fg=typer.colors.RED, err=True)
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
physical = PhysicalSchema.from_yaml(phys_file)
|
physical = PhysicalSchema.from_yaml(phys_file)
|
||||||
annotations = Annotations.from_yaml(annotations_path(cfg))
|
annotations_file = annotations_path(cfg)
|
||||||
|
annotations = Annotations.from_yaml(annotations_file)
|
||||||
records = schema_records(physical, annotations)
|
records = schema_records(physical, annotations)
|
||||||
from tht.adapters.factory import build_vector_store
|
try:
|
||||||
|
stats = sync_canonical_records(
|
||||||
stats = sync_canonical_records(
|
"schema_records",
|
||||||
"schema_records",
|
records,
|
||||||
records,
|
store=build_vector_store(cfg, require_write=True),
|
||||||
store=build_vector_store(cfg, require_write=True),
|
embedder=make_embedder(cfg.embeddings),
|
||||||
embedder=make_embedder(cfg.embeddings),
|
)
|
||||||
)
|
except VectorStoreError as exc:
|
||||||
|
code = str(exc)
|
||||||
|
error = "semantic index incompatible" if code == "semantic_index_incompatible" else "schema indexing failed"
|
||||||
|
if json_output:
|
||||||
|
_emit_json({
|
||||||
|
"code": code,
|
||||||
|
"error": error,
|
||||||
|
"operation": "index_schema",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "failed",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
typer.secho(f"ERRORE: {error}", fg=typer.colors.RED, err=True)
|
||||||
|
raise typer.Exit(code=1) from None
|
||||||
|
if json_output:
|
||||||
|
_emit_json({
|
||||||
|
"artifactIdentities": [
|
||||||
|
{"digest": _artifact_digest(annotations_file), "kind": "schema_annotations"},
|
||||||
|
{"digest": _artifact_digest(phys_file), "kind": "physical_schema"},
|
||||||
|
],
|
||||||
|
"code": "ok",
|
||||||
|
"collection": cfg.vectors.collection,
|
||||||
|
"counts": {
|
||||||
|
"added": stats.added,
|
||||||
|
"columns": sum(len(table.columns) for table in physical.tables.values()),
|
||||||
|
"deleted": stats.deleted,
|
||||||
|
"records": len(records),
|
||||||
|
"tables": len(physical.tables),
|
||||||
|
"unchanged": stats.unchanged,
|
||||||
|
"updated": stats.updated,
|
||||||
|
},
|
||||||
|
"operation": "index_schema",
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"status": "succeeded",
|
||||||
|
"workspaceId": cfg._workspace_id,
|
||||||
|
"workspaceRevision": cfg._workspace_revision,
|
||||||
|
})
|
||||||
|
return
|
||||||
_print_stats(stats)
|
_print_stats(stats)
|
||||||
|
|||||||
@@ -222,6 +222,7 @@ class QdrantConfig(BaseModel):
|
|||||||
type: Literal["qdrant"]
|
type: Literal["qdrant"]
|
||||||
base_url: str
|
base_url: str
|
||||||
collection: str = Field(min_length=1)
|
collection: str = Field(min_length=1)
|
||||||
|
collection_lifecycle: Literal["self_heal", "require_existing"] = "self_heal"
|
||||||
|
|
||||||
|
|
||||||
VectorResourceConfig = Annotated[
|
VectorResourceConfig = Annotated[
|
||||||
|
|||||||
Reference in New Issue
Block a user