mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: pristine harness JSON interfaces and require-existing semantic mode (P2)

Browse Source
This commit is contained in:
marcopan
2026-08-11 18:40:11 +02:00
parent 3cfc8c53e6
commit ca391ba59c
12 changed files with 1204 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tests/test_preprocess_cli.py
+188 -43
View File
@@ -1,4 +1,5 @@
import json
from pathlib import Path
from types import SimpleNamespace
from typer.testing import CliRunner
@@ -6,68 +7,152 @@ from typer.testing import CliRunner
from tht.cli import app
def _runtime_config(tmp_path: Path, name: str = "workspace.yaml") -> Path:
path = tmp_path / name
(tmp_path / "evidence").mkdir(exist_ok=True)
path.write_text(
f"""
runtime_identity:
workspace_id: psd-clinical
workspace_revision: {'a' * 40}
dwh:
type: postgres_direct
connection: {{database: analytics, schema: mart, user: reader, password: secret}}
vectors:
type: qdrant
base_url: http://qdrant:6333
collection: psd-clinical
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
model: qwen3-embedding:0.6b
dim: 1024
evidence:
sources:
- type: filesystem
root: {tmp_path / 'evidence'}
roots:
sessions: {tmp_path / 'sessions'}
artifacts: {tmp_path / 'artifacts'}
indexes: {tmp_path / 'indexes'}
"""
)
return path
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
result = SimpleNamespace(model_dump=lambda mode=None: {
"status": "succeeded", "generation": "gen:abc", "published": True
})
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
response = CliRunner().invoke(
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
config = _runtime_config(tmp_path)
result = SimpleNamespace(
model_dump=lambda mode=None: {
"status": "succeeded",
"generation": "gen:abc",
"published": True,
"counts": {"changed": 0, "unchanged": 0, "removed": 0, "documents": 0, "chunks": 0},
"changed": [],
"unchanged": [],
"removed": [],
"manifest_id": "manifest-1",
"run_id": "a" * 32,
"resumed_from": None,
}
)
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
assert response.exit_code == 0, response.output
assert json.loads(response.output)["generation"] == "gen:abc"
assert response.stderr == ""
assert json.loads(response.stdout) == {
"changed": [],
"code": "ok",
"counts": {"changed": 0, "chunks": 0, "documents": 0, "removed": 0, "unchanged": 0},
"generation": "gen:abc",
"manifest_id": "manifest-1",
"operation": "preprocess_evidence",
"published": True,
"removed": [],
"resumed_from": None,
"run_id": "a" * 32,
"schemaVersion": 1,
"status": "succeeded",
"unchanged": [],
"workspaceId": "psd-clinical",
"workspaceRevision": "a" * 40,
}
def test_preprocess_failure_is_structured_and_nonzero(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
monkeypatch.setattr(command, "run_from_config", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("secret detail")))
response = CliRunner().invoke(
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
config = _runtime_config(tmp_path)
monkeypatch.setattr(
command,
"run_from_config",
lambda *a, **k: (_ for _ in ()).throw(RuntimeError("secret detail")),
)
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
assert response.exit_code != 0
assert json.loads(response.output) == {"status": "failed", "error": "preprocessing failed"}
payload = json.loads(response.stdout)
assert payload == {
"code": "preprocessing_failed",
"error": "preprocessing failed",
"operation": "preprocess_evidence",
"schemaVersion": 1,
"status": "failed",
"workspaceId": "psd-clinical",
"workspaceRevision": "a" * 40,
}
assert "secret detail" not in response.output
def test_preprocess_failed_job_report_is_sanitized_json_and_nonzero(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
result = SimpleNamespace(model_dump=lambda mode=None: {
"status": "failed", "run_id": "a" * 32, "published": False,
"generation": "gen:" + "b" * 32, "changed": ["fs:one"],
})
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
response = CliRunner().invoke(
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
config = _runtime_config(tmp_path)
result = SimpleNamespace(
model_dump=lambda mode=None: {
"status": "failed",
"run_id": "a" * 32,
"published": False,
"generation": "gen:" + "b" * 32,
"changed": ["fs:one"],
"unchanged": [],
"removed": [],
"counts": {"changed": 1, "unchanged": 0, "removed": 0, "documents": 1, "chunks": 1},
"manifest_id": "manifest-1",
"resumed_from": None,
}
)
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
assert response.exit_code == 1
payload = json.loads(response.output)
payload = json.loads(response.stdout)
assert payload["status"] == "failed"
assert payload["error"] == "preprocessing job failed"
assert payload["workspaceId"] == "psd-clinical"
assert "traceback" not in response.output.lower()
def test_preprocess_real_failed_stage_result_exits_nonzero(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
from test_corpus_pipeline import Source, item, pipeline
import tht.cli.preprocess_cmd as command
config = _runtime_config(tmp_path)
result = pipeline(
tmp_path, Source([(item("one", "a"), RuntimeError("SENSITIVE EVIDENCE secret"))])
tmp_path,
Source([(item("one", "a"), RuntimeError("SENSITIVE EVIDENCE secret"))]),
).run_as_job(
workspace_id="demo", workspace_root=tmp_path,
workspace_id="demo",
workspace_root=tmp_path,
config_fingerprint="sha256:" + "1" * 64,
input_fingerprint="sha256:" + "2" * 64,
)
assert result.status == "failed"
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
response = CliRunner().invoke(
app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
)
response = CliRunner().invoke(app, ["preprocess", "evidence", "--json", "-c", str(config)])
assert response.exit_code == 1
assert json.loads(response.output)["status"] == "failed"
assert json.loads(response.stdout)["status"] == "failed"
assert "SENSITIVE EVIDENCE" not in response.output
assert "secret" not in response.output
@@ -75,19 +160,24 @@ def test_preprocess_real_failed_stage_result_exits_nonzero(monkeypatch, tmp_path
def test_preprocess_evidence_text_uses_uncapped_result_counts(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
result = SimpleNamespace(model_dump=lambda mode=None: {
"status": "succeeded", "run_id": "a" * 32,
"generation": "gen:" + "b" * 64, "published": True,
"changed": ["fs:item"] * 100,
"unchanged": ["fs:item"] * 100,
"removed": ["fs:item"] * 100,
"counts": {"changed": 1001, "unchanged": 902, "removed": 803},
})
config = _runtime_config(tmp_path)
result = SimpleNamespace(
model_dump=lambda mode=None: {
"status": "succeeded",
"run_id": "a" * 32,
"generation": "gen:" + "b" * 64,
"published": True,
"changed": ["fs:item"] * 100,
"unchanged": ["fs:item"] * 100,
"removed": ["fs:item"] * 100,
"counts": {"changed": 1001, "unchanged": 902, "removed": 803},
"manifest_id": "manifest-1",
"resumed_from": None,
}
)
monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
response = CliRunner().invoke(
app, ["preprocess", "evidence", "-c", str(tmp_path / "workspace.yaml")]
)
response = CliRunner().invoke(app, ["preprocess", "evidence", "-c", str(config)])
assert response.exit_code == 0, response.output
assert "changed=1001 unchanged=902 removed=803" in response.output
@@ -96,6 +186,7 @@ def test_preprocess_evidence_text_uses_uncapped_result_counts(monkeypatch, tmp_p
def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
config = _runtime_config(tmp_path)
called = False
def forbidden(*args, **kwargs):
@@ -106,26 +197,80 @@ def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatc
response = CliRunner().invoke(
app,
[
"preprocess", "evidence", "--resume", "gen:" + "a" * 32,
"--json", "-c", str(tmp_path / "workspace.yaml"),
"preprocess",
"evidence",
"--resume",
"gen:" + "a" * 32,
"--json",
"-c",
str(config),
],
)
assert response.exit_code != 0
assert json.loads(response.output) == {
"status": "failed", "error": "resume requires a preprocessing run id"
"code": "invalid_resume",
"error": "resume requires a preprocessing run id",
"operation": "preprocess_evidence",
"schemaVersion": 1,
"status": "failed",
}
assert called is False
def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
config = _runtime_config(tmp_path, name="3")
calls = {}
class FakePipeline:
def __init__(
self,
*,
store,
sources,
embedder,
vector_store,
embedding_model,
embedding_dimensions,
chunk_policy,
pipeline_version,
retain_published_generations,
):
calls["init"] = {
"embedding_model": embedding_model,
"embedding_dimensions": embedding_dimensions,
"pipeline_version": pipeline_version,
}
def run_as_job(self, **kwargs):
calls["run_as_job"] = kwargs
return SimpleNamespace(model_dump=lambda mode=None: {"status": "succeeded"})
monkeypatch.setattr("tht.adapters.factory.build_evidence_sources", lambda cfg: [])
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: object())
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda cfg: object())
monkeypatch.setattr("tht.corpus.pipeline.CorpusPipeline", FakePipeline)
command.run_from_config(config)
assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
assert calls["run_as_job"]["input_fingerprint"] != calls["run_as_job"]["config_fingerprint"]
def test_preprocess_evidence_gc_json_is_pristine(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
config = _runtime_config(tmp_path)
monkeypatch.setattr(command, "gc_from_config", lambda *a, **k: {
"status": "succeeded", "dry_run": True, "evicted": [], "failures": [],
"status": "succeeded",
"dry_run": True,
"evicted": [],
"failures": [],
})
response = CliRunner().invoke(
app, ["preprocess", "evidence", "gc", "--dry-run", "--json", "-c",
str(tmp_path / "workspace.yaml")]
app,
["preprocess", "evidence", "gc", "--dry-run", "--json", "-c", str(config)],
)
assert response.exit_code == 0, response.output
assert json.loads(response.output)["dry_run"] is True