fix(evidence): validate curated runtime corpus

This commit is contained in:
2026-08-25 01:59:27 +02:00
parent 9f0184388d
commit 0caa747914
11 changed files with 107 additions and 14 deletions
+54 -1
View File
@@ -2,14 +2,18 @@ import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from typer.testing import CliRunner
from tht.cli import app
def _runtime_config(tmp_path: Path, name: str = "workspace.yaml") -> Path:
def _runtime_config(
tmp_path: Path, name: str = "workspace.yaml", evidence_schema_version: int | None = None,
) -> Path:
path = tmp_path / name
(tmp_path / "evidence").mkdir(exist_ok=True)
evidence_version = "" if evidence_schema_version is None else f"\n schema_version: {evidence_schema_version}"
path.write_text(
f"""
runtime_identity:
@@ -28,6 +32,7 @@ embeddings:
model: qwen3-embedding:0.6b
dim: 1024
evidence:
{evidence_version}
sources:
- type: filesystem
root: {tmp_path / 'evidence'}
@@ -40,6 +45,54 @@ roots:
return path
def test_v2_invalid_materialized_corpus_stops_before_vector_store_construction(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
from tht.evidence import ValidationFinding, ValidationReport
config = _runtime_config(tmp_path, evidence_schema_version=2)
vector_store_constructed = False
monkeypatch.setattr(
"tht.evidence.validate_workspace_evidence",
lambda root: ValidationReport((ValidationFinding(
"error", "manifest_missing", "manifest.yaml", "missing",
),)),
)
def forbidden_vector_store(*args, **kwargs):
nonlocal vector_store_constructed
vector_store_constructed = True
raise AssertionError("invalid corpus must not reach vector upsert setup")
monkeypatch.setattr("tht.adapters.factory.build_vector_store", forbidden_vector_store)
with pytest.raises(RuntimeError, match="curated Evidence corpus is invalid"):
command.run_from_config(config)
assert vector_store_constructed is False
def test_v2_valid_materialized_corpus_is_validated_before_preprocessing(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
from tht.evidence import ValidationReport
config = _runtime_config(tmp_path, evidence_schema_version=2)
calls = []
class FakePipeline:
def run_as_job(self, **kwargs):
calls.append(("run", kwargs))
return "completed"
monkeypatch.setattr("tht.evidence.validate_workspace_evidence", lambda root: calls.append(("validate", root)) or ValidationReport(()))
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: calls.append(("vector", require_write)) or object())
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda cfg: object())
monkeypatch.setattr("tht.evidence.build_sources", lambda evidence: [])
monkeypatch.setattr("tht.evidence.build_preprocessing_pipeline", lambda **kwargs: FakePipeline())
assert command.run_from_config(config) == "completed"
assert calls[:2] == [("validate", tmp_path), ("vector", True)]
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
+13
View File
@@ -75,6 +75,18 @@ def _candidate_evaluator(cfg, *, vector_store, embedder):
return evaluate
def _validate_materialized_curated_corpus(cfg) -> None:
"""Fail closed on a v2 pinned filesystem corpus before any vector write is possible."""
if cfg.evidence is None or cfg.evidence.schema_version != 2:
return
if not any(source.type == "filesystem" for source in cfg.evidence.sources):
return
from tht.evidence import validate_workspace_evidence
if not validate_workspace_evidence(_evaluation_workspace_root(cfg)).publishable:
raise RuntimeError("curated Evidence corpus is invalid")
def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict:
value = {
**payload,
@@ -158,6 +170,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
cfg = _load_config_or_exit(config)
if cfg.embeddings is None:
raise RuntimeError("embeddings are not configured")
_validate_materialized_curated_corpus(cfg)
corpus_root = cfg.paths.artifacts.parent / "corpus"
vector_store = build_vector_store(cfg, require_write=True)
embedder = make_embedder(cfg.embeddings)
+2
View File
@@ -473,6 +473,8 @@ EvidenceSourceConfig = Annotated[
class EvidenceSourcesConfig(BaseModel):
# Version 2 is the materialized source/curated authoring layout.
schema_version: Literal[1, 2] = 1
# Legacy curated-tree configuration remains accepted during migration.
source_root: Path | None = None
# cartella curata a mano nell'ETL (relativa a source_root): unica fonte delle