fix(dwh): anchor generation operations to lease fd
This commit is contained in:
@@ -445,6 +445,8 @@ def test_unsafe_lsh_filename_is_rejected(tmp_path):
|
||||
|
||||
|
||||
def test_active_fsync_failure_restores_previous_pointer(monkeypatch, tmp_path):
|
||||
import os
|
||||
import tht.jobs.dwh_pipeline as module
|
||||
def build(physical, output):
|
||||
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json"):
|
||||
(output / name).write_text(name)
|
||||
@@ -455,27 +457,147 @@ def test_active_fsync_failure_restores_previous_pointer(monkeypatch, tmp_path):
|
||||
introspect=lambda output: output.write_text("old"), build_lsh=build,
|
||||
)
|
||||
first = first_pipeline.run()
|
||||
original_fsync = first_pipeline._fsync
|
||||
root = tmp_path / ".tht-dwh"
|
||||
root_identity = (root.stat().st_dev, root.stat().st_ino)
|
||||
original_fsync = module.os.fsync
|
||||
failed_once = False
|
||||
|
||||
def fail_active_once(path):
|
||||
def fail_active_once(fd):
|
||||
nonlocal failed_once
|
||||
if path.name == ".tht-dwh" and not failed_once:
|
||||
info = os.fstat(fd)
|
||||
if (
|
||||
(info.st_dev, info.st_ino) == root_identity
|
||||
and "ACTIVE" in os.listdir(fd)
|
||||
and not failed_once
|
||||
):
|
||||
failed_once = True
|
||||
raise OSError("injected directory fsync failure")
|
||||
original_fsync(path)
|
||||
original_fsync(fd)
|
||||
|
||||
second = DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text("new"), build_lsh=build,
|
||||
)
|
||||
monkeypatch.setattr(second, "_fsync", fail_active_once)
|
||||
monkeypatch.setattr(module.os, "fsync", fail_active_once)
|
||||
failed = second.run()
|
||||
assert failed.status == "failed"
|
||||
assert (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip() == first.run_id
|
||||
|
||||
|
||||
def test_snapshot_root_swap_after_lease_never_reads_replacement(monkeypatch, tmp_path):
|
||||
import tht.jobs.dwh_pipeline as module
|
||||
|
||||
pipeline = DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text("trusted"),
|
||||
build_lsh=lambda physical, output: [
|
||||
(output / name).write_text("trusted")
|
||||
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
||||
],
|
||||
)
|
||||
first = pipeline.run()
|
||||
assert first.status == "succeeded"
|
||||
root = tmp_path / ".tht-dwh"
|
||||
moved = tmp_path / "moved-read-root"
|
||||
replacement = root
|
||||
real_read = module._read_owned_at
|
||||
swapped = False
|
||||
|
||||
def swapping_read(directory_fd, name, *, readonly):
|
||||
nonlocal swapped
|
||||
if name == "ACTIVE" and not swapped:
|
||||
swapped = True
|
||||
replacement.rename(moved)
|
||||
replacement.mkdir(mode=0o700)
|
||||
(replacement / "sentinel").write_text("replacement-secret")
|
||||
return real_read(directory_fd, name, readonly=readonly)
|
||||
|
||||
monkeypatch.setattr(module, "_read_owned_at", swapping_read)
|
||||
try:
|
||||
with lease_dwh_snapshot(snapshot_config(tmp_path)) as snapshot:
|
||||
assert snapshot.physical.read_text() == "trusted"
|
||||
except Exception as error:
|
||||
assert "ACTIVE" in str(error) or "root" in str(error)
|
||||
assert swapped
|
||||
assert (replacement / "sentinel").read_text() == "replacement-secret"
|
||||
|
||||
|
||||
def test_publish_root_swap_after_lease_never_writes_replacement(monkeypatch, tmp_path):
|
||||
import tht.jobs.dwh_pipeline as module
|
||||
|
||||
def make(content):
|
||||
return DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text(content),
|
||||
build_lsh=lambda physical, output: [
|
||||
(output / name).write_text(content)
|
||||
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
||||
],
|
||||
)
|
||||
|
||||
first = make("old").run()
|
||||
assert first.status == "succeeded", first
|
||||
root = tmp_path / ".tht-dwh"
|
||||
moved = tmp_path / "moved-publish-root"
|
||||
real_replace = module.os.replace
|
||||
swapped = False
|
||||
|
||||
def swapping_replace(source, destination, *args, **kwargs):
|
||||
nonlocal swapped
|
||||
if destination == "ACTIVE" and kwargs.get("dst_dir_fd") is not None and not swapped:
|
||||
swapped = True
|
||||
root.rename(moved)
|
||||
root.mkdir(mode=0o700)
|
||||
(root / "sentinel").write_text("replacement-safe")
|
||||
return real_replace(source, destination, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(module.os, "replace", swapping_replace)
|
||||
result = make("new").run()
|
||||
assert result.status in {"succeeded", "failed"}
|
||||
assert swapped
|
||||
assert (root / "sentinel").read_text() == "replacement-safe"
|
||||
moved_active = (moved / "ACTIVE").read_text().strip()
|
||||
assert len(moved_active) == 32
|
||||
assert (moved / "generations" / moved_active).is_dir()
|
||||
|
||||
|
||||
def test_cleanup_root_swap_after_lease_never_deletes_replacement(monkeypatch, tmp_path):
|
||||
import tht.jobs.dwh_pipeline as module
|
||||
|
||||
pipeline = DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text("trusted"),
|
||||
build_lsh=lambda physical, output: [
|
||||
(output / name).write_text("trusted")
|
||||
for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
|
||||
],
|
||||
retain_generations=1,
|
||||
)
|
||||
pipeline.run()
|
||||
root = tmp_path / ".tht-dwh"
|
||||
moved = tmp_path / "moved-cleanup-root"
|
||||
real_open = module.os.open
|
||||
swapped = False
|
||||
|
||||
def swapping_open(path, flags, *args, **kwargs):
|
||||
nonlocal swapped
|
||||
if path == "generations" and kwargs.get("dir_fd") is not None and not swapped:
|
||||
swapped = True
|
||||
root.rename(moved)
|
||||
root.mkdir(mode=0o700)
|
||||
(root / "sentinel").write_text("replacement-safe")
|
||||
return real_open(path, flags, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(module.os, "open", swapping_open)
|
||||
pipeline._cleanup_generations()
|
||||
assert swapped
|
||||
assert (root / "sentinel").read_text() == "replacement-safe"
|
||||
|
||||
|
||||
def test_snapshot_stays_on_one_generation_across_publish(tmp_path):
|
||||
def pipeline(content):
|
||||
return DwhPreprocessPipeline(
|
||||
|
||||
Reference in New Issue
Block a user