fix(preprocess): lease DWH generation reads

This commit is contained in:
2026-07-12 05:58:12 +02:00
parent db35ddb041
commit 6bb158233f
3 changed files with 205 additions and 23 deletions
+65
View File
@@ -7,6 +7,7 @@ from typer.testing import CliRunner
from tht.cli import app
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
from tht.jobs.dwh_pipeline import lease_dwh_snapshot
from tht.jobs.locking import _lock_name
@@ -228,3 +229,67 @@ def test_generation_retention_keeps_active_and_one_rollback(tmp_path):
run_ids.append(report.run_id)
remaining = {path.name for path in (tmp_path / ".tht-dwh" / "generations").iterdir()}
assert remaining == set(run_ids[-2:])
def test_reader_lease_blocks_retain_one_publisher_until_file_reads_finish(tmp_path):
import threading
import time
from types import SimpleNamespace
def make(content, retain=1):
return DwhPreprocessPipeline(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint=FP, input_fingerprint=FP,
introspect=lambda output: output.write_text(content),
build_lsh=lambda physical, output: [
(output / name).write_text(content)
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
], retain_generations=retain,
)
first = make("old").run()
cfg = SimpleNamespace(paths=SimpleNamespace(
artifacts=tmp_path / "artifacts", indexes=tmp_path / "indexes"
))
completed = threading.Event()
with lease_dwh_snapshot(cfg) as snapshot:
thread = threading.Thread(target=lambda: (make("new").run(), completed.set()))
thread.start()
time.sleep(0.05)
assert not completed.is_set()
assert snapshot.physical.read_text() == "old"
assert snapshot.generation == first.run_id
thread.join(timeout=2)
assert completed.is_set()
assert not (tmp_path / ".tht-dwh" / "generations" / first.run_id).exists()
def test_cleanup_never_follows_top_level_or_child_symlinks(tmp_path):
external = tmp_path / "external"
external.mkdir()
victim = external / "victim"
victim.write_text("safe")
generations = tmp_path / ".tht-dwh" / "generations"
generations.mkdir(parents=True)
(generations / ("a" * 32)).symlink_to(external, target_is_directory=True)
def make(content, retain=1):
return DwhPreprocessPipeline(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint=FP, input_fingerprint=FP,
introspect=lambda output: output.write_text(content),
build_lsh=lambda physical, output: [
(output / name).write_text(content)
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
], retain_generations=retain,
)
first = make("one", retain=2).run()
make("two", retain=2).run()
old = generations / first.run_id
old.chmod(0o700)
(old / "hostile-link").symlink_to(victim)
make("three").run()
assert victim.read_text() == "safe"
assert victim.stat().st_mode & 0o200