fix(dwh): anchor generation operations to lease fd

This commit is contained in:
2026-07-12 07:26:10 +02:00
parent 24e61d5713
commit 2f6daaaea6
3 changed files with 495 additions and 242 deletions
+127 -5
View File
@@ -445,6 +445,8 @@ def test_unsafe_lsh_filename_is_rejected(tmp_path):
def test_active_fsync_failure_restores_previous_pointer(monkeypatch, tmp_path):
import os
import tht.jobs.dwh_pipeline as module
def build(physical, output):
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json"):
(output / name).write_text(name)
@@ -455,27 +457,147 @@ def test_active_fsync_failure_restores_previous_pointer(monkeypatch, tmp_path):
introspect=lambda output: output.write_text("old"), build_lsh=build,
)
first = first_pipeline.run()
original_fsync = first_pipeline._fsync
root = tmp_path / ".tht-dwh"
root_identity = (root.stat().st_dev, root.stat().st_ino)
original_fsync = module.os.fsync
failed_once = False
def fail_active_once(path):
def fail_active_once(fd):
nonlocal failed_once
if path.name == ".tht-dwh" and not failed_once:
info = os.fstat(fd)
if (
(info.st_dev, info.st_ino) == root_identity
and "ACTIVE" in os.listdir(fd)
and not failed_once
):
failed_once = True
raise OSError("injected directory fsync failure")
original_fsync(path)
original_fsync(fd)
second = DwhPreprocessPipeline(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint=FP, input_fingerprint=FP,
introspect=lambda output: output.write_text("new"), build_lsh=build,
)
monkeypatch.setattr(second, "_fsync", fail_active_once)
monkeypatch.setattr(module.os, "fsync", fail_active_once)
failed = second.run()
assert failed.status == "failed"
assert (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip() == first.run_id
def test_snapshot_root_swap_after_lease_never_reads_replacement(monkeypatch, tmp_path):
import tht.jobs.dwh_pipeline as module
pipeline = DwhPreprocessPipeline(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint=FP, input_fingerprint=FP,
introspect=lambda output: output.write_text("trusted"),
build_lsh=lambda physical, output: [
(output / name).write_text("trusted")
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
],
)
first = pipeline.run()
assert first.status == "succeeded"
root = tmp_path / ".tht-dwh"
moved = tmp_path / "moved-read-root"
replacement = root
real_read = module._read_owned_at
swapped = False
def swapping_read(directory_fd, name, *, readonly):
nonlocal swapped
if name == "ACTIVE" and not swapped:
swapped = True
replacement.rename(moved)
replacement.mkdir(mode=0o700)
(replacement / "sentinel").write_text("replacement-secret")
return real_read(directory_fd, name, readonly=readonly)
monkeypatch.setattr(module, "_read_owned_at", swapping_read)
try:
with lease_dwh_snapshot(snapshot_config(tmp_path)) as snapshot:
assert snapshot.physical.read_text() == "trusted"
except Exception as error:
assert "ACTIVE" in str(error) or "root" in str(error)
assert swapped
assert (replacement / "sentinel").read_text() == "replacement-secret"
def test_publish_root_swap_after_lease_never_writes_replacement(monkeypatch, tmp_path):
import tht.jobs.dwh_pipeline as module
def make(content):
return DwhPreprocessPipeline(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint=FP, input_fingerprint=FP,
introspect=lambda output: output.write_text(content),
build_lsh=lambda physical, output: [
(output / name).write_text(content)
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
],
)
first = make("old").run()
assert first.status == "succeeded", first
root = tmp_path / ".tht-dwh"
moved = tmp_path / "moved-publish-root"
real_replace = module.os.replace
swapped = False
def swapping_replace(source, destination, *args, **kwargs):
nonlocal swapped
if destination == "ACTIVE" and kwargs.get("dst_dir_fd") is not None and not swapped:
swapped = True
root.rename(moved)
root.mkdir(mode=0o700)
(root / "sentinel").write_text("replacement-safe")
return real_replace(source, destination, *args, **kwargs)
monkeypatch.setattr(module.os, "replace", swapping_replace)
result = make("new").run()
assert result.status in {"succeeded", "failed"}
assert swapped
assert (root / "sentinel").read_text() == "replacement-safe"
moved_active = (moved / "ACTIVE").read_text().strip()
assert len(moved_active) == 32
assert (moved / "generations" / moved_active).is_dir()
def test_cleanup_root_swap_after_lease_never_deletes_replacement(monkeypatch, tmp_path):
import tht.jobs.dwh_pipeline as module
pipeline = DwhPreprocessPipeline(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint=FP, input_fingerprint=FP,
introspect=lambda output: output.write_text("trusted"),
build_lsh=lambda physical, output: [
(output / name).write_text("trusted")
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
],
retain_generations=1,
)
pipeline.run()
root = tmp_path / ".tht-dwh"
moved = tmp_path / "moved-cleanup-root"
real_open = module.os.open
swapped = False
def swapping_open(path, flags, *args, **kwargs):
nonlocal swapped
if path == "generations" and kwargs.get("dir_fd") is not None and not swapped:
swapped = True
root.rename(moved)
root.mkdir(mode=0o700)
(root / "sentinel").write_text("replacement-safe")
return real_open(path, flags, *args, **kwargs)
monkeypatch.setattr(module.os, "open", swapping_open)
pipeline._cleanup_generations()
assert swapped
assert (root / "sentinel").read_text() == "replacement-safe"
def test_snapshot_stays_on_one_generation_across_publish(tmp_path):
def pipeline(content):
return DwhPreprocessPipeline(