fix(preprocess): lease DWH generation reads
This commit is contained in:
@@ -7,6 +7,7 @@ from typer.testing import CliRunner
|
||||
from tht.cli import app
|
||||
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
from tht.jobs.dwh_pipeline import lease_dwh_snapshot
|
||||
from tht.jobs.locking import _lock_name
|
||||
|
||||
|
||||
@@ -228,3 +229,67 @@ def test_generation_retention_keeps_active_and_one_rollback(tmp_path):
|
||||
run_ids.append(report.run_id)
|
||||
remaining = {path.name for path in (tmp_path / ".tht-dwh" / "generations").iterdir()}
|
||||
assert remaining == set(run_ids[-2:])
|
||||
|
||||
|
||||
def test_reader_lease_blocks_retain_one_publisher_until_file_reads_finish(tmp_path):
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
def make(content, retain=1):
|
||||
return DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text(content),
|
||||
build_lsh=lambda physical, output: [
|
||||
(output / name).write_text(content)
|
||||
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
||||
], retain_generations=retain,
|
||||
)
|
||||
|
||||
first = make("old").run()
|
||||
cfg = SimpleNamespace(paths=SimpleNamespace(
|
||||
artifacts=tmp_path / "artifacts", indexes=tmp_path / "indexes"
|
||||
))
|
||||
completed = threading.Event()
|
||||
with lease_dwh_snapshot(cfg) as snapshot:
|
||||
thread = threading.Thread(target=lambda: (make("new").run(), completed.set()))
|
||||
thread.start()
|
||||
time.sleep(0.05)
|
||||
assert not completed.is_set()
|
||||
assert snapshot.physical.read_text() == "old"
|
||||
assert snapshot.generation == first.run_id
|
||||
thread.join(timeout=2)
|
||||
assert completed.is_set()
|
||||
assert not (tmp_path / ".tht-dwh" / "generations" / first.run_id).exists()
|
||||
|
||||
|
||||
def test_cleanup_never_follows_top_level_or_child_symlinks(tmp_path):
|
||||
external = tmp_path / "external"
|
||||
external.mkdir()
|
||||
victim = external / "victim"
|
||||
victim.write_text("safe")
|
||||
|
||||
generations = tmp_path / ".tht-dwh" / "generations"
|
||||
generations.mkdir(parents=True)
|
||||
(generations / ("a" * 32)).symlink_to(external, target_is_directory=True)
|
||||
|
||||
def make(content, retain=1):
|
||||
return DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text(content),
|
||||
build_lsh=lambda physical, output: [
|
||||
(output / name).write_text(content)
|
||||
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
||||
], retain_generations=retain,
|
||||
)
|
||||
|
||||
first = make("one", retain=2).run()
|
||||
make("two", retain=2).run()
|
||||
old = generations / first.run_id
|
||||
old.chmod(0o700)
|
||||
(old / "hostile-link").symlink_to(victim)
|
||||
make("three").run()
|
||||
assert victim.read_text() == "safe"
|
||||
assert victim.stat().st_mode & 0o200
|
||||
|
||||
Reference in New Issue
Block a user