fix(preprocess): bind DWH artifacts to workspace
This commit is contained in:
@@ -14,6 +14,18 @@ from tht.jobs.locking import _lock_name
|
||||
FP = "sha256:" + hashlib.sha256(b"test").hexdigest()
|
||||
|
||||
|
||||
def snapshot_config(tmp_path, workspace_id="demo"):
|
||||
from types import SimpleNamespace
|
||||
|
||||
cfg = SimpleNamespace(
|
||||
paths=SimpleNamespace(artifacts=tmp_path / "artifacts", indexes=tmp_path / "indexes"),
|
||||
_workspace_id=workspace_id,
|
||||
_config_source="test",
|
||||
)
|
||||
cfg.model_dump_json = lambda: "test"
|
||||
return cfg
|
||||
|
||||
|
||||
def test_dwh_and_evidence_jobs_have_distinct_lock_names():
|
||||
assert _lock_name("demo", "dwh") != _lock_name("demo", "evidence")
|
||||
|
||||
@@ -43,6 +55,68 @@ def test_selected_dwh_stages_run_in_declared_order(tmp_path):
|
||||
]
|
||||
|
||||
|
||||
def test_shared_root_rejects_other_workspace_before_builder_or_read(tmp_path):
|
||||
calls = []
|
||||
owner = DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text("catalog"),
|
||||
build_lsh=lambda physical, output: _write_lsh([], physical, output),
|
||||
)
|
||||
published = owner.run()
|
||||
contender = DwhPreprocessPipeline(
|
||||
workspace_id="other", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: calls.append("introspect"),
|
||||
build_lsh=lambda physical, output: calls.append("lsh"),
|
||||
)
|
||||
|
||||
import pytest
|
||||
with pytest.raises(Exception, match="different workspace configuration"):
|
||||
contender.run()
|
||||
with pytest.raises(Exception, match="different workspace configuration"):
|
||||
resolve_dwh_snapshot(snapshot_config(tmp_path, "other"))
|
||||
|
||||
assert calls == []
|
||||
assert resolve_dwh_snapshot(snapshot_config(tmp_path)).generation == published.run_id
|
||||
|
||||
|
||||
def test_shared_root_mismatch_fails_without_deadlock_while_owner_reader_is_active(tmp_path):
|
||||
import threading
|
||||
|
||||
owner = DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: output.write_text("catalog"),
|
||||
build_lsh=lambda physical, output: _write_lsh([], physical, output),
|
||||
)
|
||||
owner.run()
|
||||
contender = DwhPreprocessPipeline(
|
||||
workspace_id="other", workspace_root=tmp_path,
|
||||
config_fingerprint=FP, input_fingerprint=FP,
|
||||
introspect=lambda output: (_ for _ in ()).throw(AssertionError("builder called")),
|
||||
build_lsh=lambda physical, output: None,
|
||||
)
|
||||
finished = threading.Event()
|
||||
errors = []
|
||||
with lease_dwh_snapshot(snapshot_config(tmp_path)):
|
||||
thread = threading.Thread(
|
||||
target=lambda: (errors.append(_capture_error(contender.run)), finished.set())
|
||||
)
|
||||
thread.start()
|
||||
assert finished.wait(2)
|
||||
thread.join(2)
|
||||
assert "different workspace configuration" in str(errors[0])
|
||||
|
||||
|
||||
def _capture_error(operation):
|
||||
try:
|
||||
operation()
|
||||
except Exception as error:
|
||||
return error
|
||||
raise AssertionError("operation unexpectedly succeeded")
|
||||
|
||||
|
||||
def _write_lsh(calls, physical: Path, output: Path):
|
||||
calls.append("lsh")
|
||||
assert physical.read_text() == "catalog"
|
||||
@@ -189,8 +263,6 @@ def test_active_fsync_failure_restores_previous_pointer(monkeypatch, tmp_path):
|
||||
|
||||
|
||||
def test_snapshot_stays_on_one_generation_across_publish(tmp_path):
|
||||
from types import SimpleNamespace
|
||||
|
||||
def pipeline(content):
|
||||
return DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
@@ -203,9 +275,7 @@ def test_snapshot_stays_on_one_generation_across_publish(tmp_path):
|
||||
)
|
||||
|
||||
first = pipeline("old").run()
|
||||
cfg = SimpleNamespace(paths=SimpleNamespace(
|
||||
artifacts=tmp_path / "artifacts", indexes=tmp_path / "indexes"
|
||||
))
|
||||
cfg = snapshot_config(tmp_path)
|
||||
snapshot = resolve_dwh_snapshot(cfg)
|
||||
pipeline("new").run()
|
||||
assert snapshot.generation == first.run_id
|
||||
@@ -328,8 +398,6 @@ def test_retention_candidate_swap_to_symlink_is_never_followed(monkeypatch, tmp_
|
||||
def test_reader_lease_blocks_retain_one_publisher_until_file_reads_finish(tmp_path):
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
def make(content, retain=1):
|
||||
return DwhPreprocessPipeline(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
@@ -342,9 +410,7 @@ def test_reader_lease_blocks_retain_one_publisher_until_file_reads_finish(tmp_pa
|
||||
)
|
||||
|
||||
first = make("old").run()
|
||||
cfg = SimpleNamespace(paths=SimpleNamespace(
|
||||
artifacts=tmp_path / "artifacts", indexes=tmp_path / "indexes"
|
||||
))
|
||||
cfg = snapshot_config(tmp_path)
|
||||
completed = threading.Event()
|
||||
with lease_dwh_snapshot(cfg) as snapshot:
|
||||
thread = threading.Thread(target=lambda: (make("new").run(), completed.set()))
|
||||
|
||||
Reference in New Issue
Block a user