142 lines
4.8 KiB
Python
142 lines
4.8 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from typer.testing import CliRunner
|
|
|
|
from tht.cli import app
|
|
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline
|
|
from tht.jobs.locking import _lock_name
|
|
|
|
|
|
FP = "sha256:" + hashlib.sha256(b"test").hexdigest()
|
|
|
|
|
|
def test_dwh_and_evidence_jobs_have_distinct_lock_names():
|
|
assert _lock_name("demo", "dwh") != _lock_name("demo", "evidence")
|
|
|
|
|
|
def test_selected_dwh_stages_run_in_declared_order(tmp_path):
|
|
calls = []
|
|
pipeline = DwhPreprocessPipeline(
|
|
workspace_id="demo",
|
|
workspace_root=tmp_path,
|
|
config_fingerprint=FP,
|
|
input_fingerprint=FP,
|
|
introspect=lambda output: (calls.append("introspect"), output.write_text("catalog")),
|
|
build_lsh=lambda physical, output: _write_lsh(calls, physical, output),
|
|
)
|
|
|
|
report = pipeline.run(("introspect", "lsh"))
|
|
|
|
assert report.status == "succeeded"
|
|
assert calls == ["introspect", "lsh"]
|
|
assert [stage.name for stage in report.stages] == ["introspect", "lsh"]
|
|
active = (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip()
|
|
published = tmp_path / ".tht-dwh" / "generations" / active
|
|
assert (published / "physical.yaml").read_text() == "catalog"
|
|
assert sorted(path.name for path in published.iterdir()) == [
|
|
"demo_lsh.pkl", "demo_meta.json", "demo_minhashes.pkl", "physical.yaml"
|
|
]
|
|
|
|
|
|
def _write_lsh(calls, physical: Path, output: Path):
|
|
calls.append("lsh")
|
|
assert physical.read_text() == "catalog"
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json"):
|
|
(output / name).write_text(name)
|
|
|
|
|
|
def test_preprocess_dwh_json_is_pristine(monkeypatch, tmp_path):
|
|
import tht.cli.preprocess_cmd as command
|
|
|
|
class Report:
|
|
status = "succeeded"
|
|
|
|
def model_dump(self, mode=None):
|
|
return {"status": "succeeded", "run_id": "a" * 32, "stages": []}
|
|
|
|
seen = {}
|
|
|
|
def run(config, *, steps, resume):
|
|
seen.update(config=config, steps=steps, resume=resume)
|
|
return Report()
|
|
|
|
monkeypatch.setattr(command, "run_dwh_from_config", run)
|
|
response = CliRunner().invoke(
|
|
app,
|
|
[
|
|
"preprocess", "dwh", "--steps", "introspect,lsh", "--json",
|
|
"-c", str(tmp_path / "workspace.yaml"),
|
|
],
|
|
)
|
|
|
|
assert response.exit_code == 0, response.output
|
|
assert json.loads(response.output)["run_id"] == "a" * 32
|
|
assert seen["steps"] == ("introspect", "lsh")
|
|
|
|
|
|
def test_preprocess_dwh_rejects_unknown_or_duplicate_steps(monkeypatch, tmp_path):
|
|
import tht.cli.preprocess_cmd as command
|
|
|
|
called = False
|
|
|
|
def forbidden(*args, **kwargs):
|
|
nonlocal called
|
|
called = True
|
|
|
|
monkeypatch.setattr(command, "run_dwh_from_config", forbidden)
|
|
runner = CliRunner()
|
|
for value in ("introspect,unknown", "lsh,lsh", ""):
|
|
response = runner.invoke(
|
|
app,
|
|
["preprocess", "dwh", "--steps", value, "--json", "-c", str(tmp_path / "w.yaml")],
|
|
)
|
|
assert response.exit_code == 2
|
|
assert json.loads(response.output)["status"] == "failed"
|
|
assert called is False
|
|
|
|
|
|
def test_failed_multi_file_build_never_replaces_active_generation(tmp_path):
|
|
def catalog(output):
|
|
output.write_text("old-catalog")
|
|
|
|
first = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=catalog,
|
|
build_lsh=lambda physical, output: [
|
|
(output / name).write_text(name)
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
|
],
|
|
).run(("introspect", "lsh"))
|
|
assert first.status == "succeeded"
|
|
old_active = (tmp_path / ".tht-dwh" / "ACTIVE").read_text()
|
|
|
|
def partial_lsh(physical, output):
|
|
(output / "demo_lsh.pkl").write_text("new-but-partial")
|
|
raise RuntimeError("crash between LSH files")
|
|
|
|
failed = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: output.write_text("new-catalog"),
|
|
build_lsh=partial_lsh,
|
|
).run(("introspect", "lsh"))
|
|
|
|
assert failed.status == "failed"
|
|
assert (tmp_path / ".tht-dwh" / "ACTIVE").read_text() == old_active
|
|
|
|
resumed = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: (_ for _ in ()).throw(
|
|
AssertionError("completed introspection must not repeat")
|
|
),
|
|
build_lsh=lambda physical, output: [
|
|
(output / name).write_text("recovered")
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
|
],
|
|
).run(("introspect", "lsh"), resume_run_id=failed.run_id)
|
|
assert resumed.status == "succeeded"
|