231 lines
8.2 KiB
Python
231 lines
8.2 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from typer.testing import CliRunner
|
|
|
|
from tht.cli import app
|
|
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline
|
|
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
|
from tht.jobs.locking import _lock_name
|
|
|
|
|
|
FP = "sha256:" + hashlib.sha256(b"test").hexdigest()
|
|
|
|
|
|
def test_dwh_and_evidence_jobs_have_distinct_lock_names():
|
|
assert _lock_name("demo", "dwh") != _lock_name("demo", "evidence")
|
|
|
|
|
|
def test_selected_dwh_stages_run_in_declared_order(tmp_path):
|
|
calls = []
|
|
pipeline = DwhPreprocessPipeline(
|
|
workspace_id="demo",
|
|
workspace_root=tmp_path,
|
|
config_fingerprint=FP,
|
|
input_fingerprint=FP,
|
|
introspect=lambda output: (calls.append("introspect"), output.write_text("catalog")),
|
|
build_lsh=lambda physical, output: _write_lsh(calls, physical, output),
|
|
)
|
|
|
|
report = pipeline.run(("introspect", "lsh"))
|
|
|
|
assert report.status == "succeeded"
|
|
assert calls == ["introspect", "lsh"]
|
|
assert [stage.name for stage in report.stages] == ["introspect", "lsh"]
|
|
active = (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip()
|
|
published = tmp_path / ".tht-dwh" / "generations" / active
|
|
assert (published / "physical.yaml").read_text() == "catalog"
|
|
assert sorted(path.name for path in published.iterdir()) == [
|
|
"demo_lsh.pkl", "demo_meta.json", "demo_minhashes.pkl",
|
|
"generation-manifest.json", "physical.yaml",
|
|
]
|
|
|
|
|
|
def _write_lsh(calls, physical: Path, output: Path):
|
|
calls.append("lsh")
|
|
assert physical.read_text() == "catalog"
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json"):
|
|
(output / name).write_text(name)
|
|
|
|
|
|
def test_preprocess_dwh_json_is_pristine(monkeypatch, tmp_path):
|
|
import tht.cli.preprocess_cmd as command
|
|
|
|
class Report:
|
|
status = "succeeded"
|
|
|
|
def model_dump(self, mode=None):
|
|
return {"status": "succeeded", "run_id": "a" * 32, "stages": []}
|
|
|
|
seen = {}
|
|
|
|
def run(config, *, steps, resume):
|
|
seen.update(config=config, steps=steps, resume=resume)
|
|
return Report()
|
|
|
|
monkeypatch.setattr(command, "run_dwh_from_config", run)
|
|
response = CliRunner().invoke(
|
|
app,
|
|
[
|
|
"preprocess", "dwh", "--steps", "introspect,lsh", "--json",
|
|
"-c", str(tmp_path / "workspace.yaml"),
|
|
],
|
|
)
|
|
|
|
assert response.exit_code == 0, response.output
|
|
assert json.loads(response.output)["run_id"] == "a" * 32
|
|
assert seen["steps"] == ("introspect", "lsh")
|
|
|
|
|
|
def test_preprocess_dwh_rejects_unknown_or_duplicate_steps(monkeypatch, tmp_path):
|
|
import tht.cli.preprocess_cmd as command
|
|
|
|
called = False
|
|
|
|
def forbidden(*args, **kwargs):
|
|
nonlocal called
|
|
called = True
|
|
|
|
monkeypatch.setattr(command, "run_dwh_from_config", forbidden)
|
|
runner = CliRunner()
|
|
for value in ("introspect,unknown", "lsh,lsh", ""):
|
|
response = runner.invoke(
|
|
app,
|
|
["preprocess", "dwh", "--steps", value, "--json", "-c", str(tmp_path / "w.yaml")],
|
|
)
|
|
assert response.exit_code == 2
|
|
assert json.loads(response.output)["status"] == "failed"
|
|
assert called is False
|
|
|
|
|
|
def test_failed_multi_file_build_never_replaces_active_generation(tmp_path):
|
|
def catalog(output):
|
|
output.write_text("old-catalog")
|
|
|
|
first = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=catalog,
|
|
build_lsh=lambda physical, output: [
|
|
(output / name).write_text(name)
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
|
],
|
|
).run(("introspect", "lsh"))
|
|
assert first.status == "succeeded"
|
|
old_active = (tmp_path / ".tht-dwh" / "ACTIVE").read_text()
|
|
|
|
def partial_lsh(physical, output):
|
|
(output / "demo_lsh.pkl").write_text("new-but-partial")
|
|
raise RuntimeError("crash between LSH files")
|
|
|
|
failed = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: output.write_text("new-catalog"),
|
|
build_lsh=partial_lsh,
|
|
).run(("introspect", "lsh"))
|
|
|
|
assert failed.status == "failed"
|
|
assert (tmp_path / ".tht-dwh" / "ACTIVE").read_text() == old_active
|
|
|
|
resumed = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: (_ for _ in ()).throw(
|
|
AssertionError("completed introspection must not repeat")
|
|
),
|
|
build_lsh=lambda physical, output: [
|
|
(output / name).write_text("recovered")
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
|
],
|
|
).run(("introspect", "lsh"), resume_run_id=failed.run_id)
|
|
assert resumed.status == "succeeded"
|
|
|
|
|
|
def test_unsafe_lsh_filename_is_rejected(tmp_path):
|
|
import pytest
|
|
|
|
with pytest.raises(ValueError, match="flat safe"):
|
|
DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: None, build_lsh=lambda physical, output: None,
|
|
lsh_filenames=("../escape.pkl", "ok.pkl", "meta.json"),
|
|
)
|
|
|
|
|
|
def test_active_fsync_failure_restores_previous_pointer(monkeypatch, tmp_path):
|
|
def build(physical, output):
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json"):
|
|
(output / name).write_text(name)
|
|
|
|
first_pipeline = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: output.write_text("old"), build_lsh=build,
|
|
)
|
|
first = first_pipeline.run()
|
|
original_fsync = first_pipeline._fsync
|
|
failed_once = False
|
|
|
|
def fail_active_once(path):
|
|
nonlocal failed_once
|
|
if path.name == ".tht-dwh" and not failed_once:
|
|
failed_once = True
|
|
raise OSError("injected directory fsync failure")
|
|
original_fsync(path)
|
|
|
|
second = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: output.write_text("new"), build_lsh=build,
|
|
)
|
|
monkeypatch.setattr(second, "_fsync", fail_active_once)
|
|
failed = second.run()
|
|
assert failed.status == "failed"
|
|
assert (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip() == first.run_id
|
|
|
|
|
|
def test_snapshot_stays_on_one_generation_across_publish(tmp_path):
|
|
from types import SimpleNamespace
|
|
|
|
def pipeline(content):
|
|
return DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output: output.write_text(content),
|
|
build_lsh=lambda physical, output: [
|
|
(output / name).write_text(content)
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
|
],
|
|
)
|
|
|
|
first = pipeline("old").run()
|
|
cfg = SimpleNamespace(paths=SimpleNamespace(
|
|
artifacts=tmp_path / "artifacts", indexes=tmp_path / "indexes"
|
|
))
|
|
snapshot = resolve_dwh_snapshot(cfg)
|
|
pipeline("new").run()
|
|
assert snapshot.generation == first.run_id
|
|
assert snapshot.physical.read_text() == "old"
|
|
assert (snapshot.lsh_dir / "demo_meta.json").read_text() == "old"
|
|
|
|
|
|
def test_generation_retention_keeps_active_and_one_rollback(tmp_path):
|
|
run_ids = []
|
|
for index in range(5):
|
|
report = DwhPreprocessPipeline(
|
|
workspace_id="demo", workspace_root=tmp_path,
|
|
config_fingerprint=FP, input_fingerprint=FP,
|
|
introspect=lambda output, i=index: output.write_text(str(i)),
|
|
build_lsh=lambda physical, output, i=index: [
|
|
(output / name).write_text(str(i))
|
|
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
|
],
|
|
retain_generations=2,
|
|
).run()
|
|
run_ids.append(report.run_id)
|
|
remaining = {path.name for path in (tmp_path / ".tht-dwh" / "generations").iterdir()}
|
|
assert remaining == set(run_ids[-2:])
|