feat(evidence): validate canonical curated corpus

2026-08-24 17:38:22 +02:00
parent 5c6228f8c2
commit ae0976a4aa
5 changed files with 1217 additions and 0 deletions
@@ -0,0 +1,281 @@
+import hashlib
+
+import pytest
+from pydantic import ValidationError
+
+from tht.evidence import (
+    CuratedEvidence,
+    EvidenceManifest,
+    dump_curated_markdown,
+    dump_manifest,
+    load_manifest,
+    validate_workspace_evidence,
+)
+
+
+def _evidence(source_text: str, *, review_items=()):
+    return CuratedEvidence.model_validate({
+        "schema_version": 1,
+        "id": "evidence:fascia-pediatrica",
+        "title": "Fascia pediatrica",
+        "kind": "domain",
+        "purposes": ["disambiguation"],
+        "applies_to": {"concepts": ["fascia pediatrica"]},
+        "language": "it",
+        "provenance": {
+            "source_file": "source/domain/patient.md",
+            "source_sha256": "sha256:" + hashlib.sha256(source_text.encode()).hexdigest(),
+            "supporting_excerpts": ["I pazienti sotto i 18 anni sono pediatrici."],
+        },
+        "review_items": list(review_items),
+        "payload": {"rule": "La fascia pediatrica comprende i minori."},
+    })
+
+
+def _write_workspace(root, evidence, source_text):
+    evidence_root = root / "evidence"
+    source_path = evidence_root / "source" / "domain" / "patient.md"
+    curated_path = evidence_root / "curated" / "domain" / "fascia-pediatrica.md"
+    source_path.parent.mkdir(parents=True)
+    curated_path.parent.mkdir(parents=True)
+    source_path.write_text(source_text, encoding="utf-8")
+    curated_path.write_text(dump_curated_markdown(evidence), encoding="utf-8")
+    (evidence_root / "manifest.yaml").write_text(dump_manifest(EvidenceManifest.model_validate({
+        "schema_version": 1,
+        "pipeline_version": "evidence-authoring-v1",
+        "sources": {
+            evidence.provenance.source_file: {
+                "sha256": evidence.provenance.source_sha256,
+                "units": [evidence.id],
+            },
+        },
+        "orphans": [],
+    })), encoding="utf-8")
+
+
+def test_workspace_validation_allows_a_coherent_curated_unit(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    _write_workspace(tmp_path, _evidence(source_text), source_text)
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is True
+    assert report.findings == ()
+
+
+def test_workspace_validation_keeps_review_items_visible_and_blocks_publication(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    _write_workspace(tmp_path, _evidence(source_text, review_items=[{
+        "code": "ambiguous_source_statement",
+        "message": "Il sorgente non chiarisce la data di riferimento.",
+        "field": "payload.rule",
+    }]), source_text)
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert [finding.code for finding in report.findings] == ["unresolved_review_item"]
+
+
+def test_manifest_round_trip_is_versioned_and_deterministic(tmp_path):
+    manifest = EvidenceManifest.model_validate({
+        "schema_version": 1,
+        "pipeline_version": "evidence-authoring-v1",
+        "sources": {
+            "source/domain/patient.md": {
+                "sha256": "sha256:" + "a" * 64,
+                "units": ["evidence:fascia-pediatrica"],
+            },
+        },
+        "orphans": [],
+    })
+    path = tmp_path / "evidence" / "manifest.yaml"
+    path.parent.mkdir(parents=True)
+    path.write_text(dump_manifest(manifest), encoding="utf-8")
+
+    loaded = load_manifest(path)
+
+    assert loaded == manifest
+    assert path.read_text(encoding="utf-8") == dump_manifest(manifest)
+
+
+def test_manifest_rejects_unknown_fields_and_incompatible_pipeline_version():
+    with pytest.raises(ValidationError):
+        EvidenceManifest.model_validate({
+            "schema_version": 1,
+            "pipeline_version": "evidence-authoring-v2",
+            "sources": {},
+            "orphans": [],
+            "approved": True,
+        })
+
+
+def test_workspace_validation_requires_a_manifest(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    _write_workspace(tmp_path, _evidence(source_text), source_text)
+    (tmp_path / "evidence" / "manifest.yaml").unlink()
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert [finding.code for finding in report.findings] == ["manifest_missing"]
+
+
+def test_workspace_validation_reports_a_non_utf8_manifest_without_raising(tmp_path):
+    evidence_root = tmp_path / "evidence"
+    evidence_root.mkdir()
+    (evidence_root / "manifest.yaml").write_bytes(b"\xff")
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert [finding.code for finding in report.findings] == ["manifest_invalid"]
+
+
+def test_workspace_validation_reports_invalid_curated_documents_without_raising(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    _write_workspace(tmp_path, _evidence(source_text), source_text)
+    (tmp_path / "evidence" / "curated" / "domain" / "fascia-pediatrica.md").write_text(
+        "not canonical frontmatter", encoding="utf-8",
+    )
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert [finding.code for finding in report.findings] == ["curated_invalid"]
+
+
+def test_workspace_validation_preserves_manifest_orphans_but_blocks_publication(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    _write_workspace(tmp_path, _evidence(source_text), source_text)
+    manifest_path = tmp_path / "evidence" / "manifest.yaml"
+    manifest = load_manifest(manifest_path).model_copy(update={"orphans": ("evidence:retired",)})
+    manifest_path.write_text(dump_manifest(manifest), encoding="utf-8")
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert [finding.code for finding in report.findings] == ["orphaned_unit"]
+
+
+def test_workspace_validation_requires_manifest_and_provenance_to_agree(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    evidence = _evidence(source_text)
+    _write_workspace(tmp_path, evidence, source_text)
+    manifest_path = tmp_path / "evidence" / "manifest.yaml"
+    manifest = EvidenceManifest.model_validate({
+        "schema_version": 1,
+        "pipeline_version": "evidence-authoring-v1",
+        "sources": {
+            evidence.provenance.source_file: {
+                "sha256": "sha256:" + "b" * 64,
+                "units": ["evidence:another-unit"],
+            },
+        },
+        "orphans": [],
+    })
+    manifest_path.write_text(dump_manifest(manifest), encoding="utf-8")
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert {finding.code for finding in report.findings} == {
+        "manifest_source_hash_mismatch",
+        "manifest_unit_missing",
+        "source_hash_mismatch",
+        "manifest_unit_without_curated",
+    }
+
+
+def test_workspace_validation_checks_unreferenced_manifest_sources_and_units(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    _write_workspace(tmp_path, _evidence(source_text), source_text)
+    manifest_path = tmp_path / "evidence" / "manifest.yaml"
+    manifest = EvidenceManifest.model_validate({
+        "schema_version": 1,
+        "pipeline_version": "evidence-authoring-v1",
+        "sources": {
+            "source/domain/missing.md": {
+                "sha256": "sha256:" + "a" * 64,
+                "units": ["evidence:removed"],
+            },
+        },
+        "orphans": [],
+    })
+    manifest_path.write_text(dump_manifest(manifest), encoding="utf-8")
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert {finding.code for finding in report.findings} == {
+        "manifest_source_missing",
+        "source_missing",
+        "manifest_unit_without_curated",
+    }
+
+
+def test_manifest_rejects_unsafe_source_paths_and_legacy_unit_identifiers():
+    with pytest.raises(ValidationError):
+        EvidenceManifest.model_validate({
+            "schema_version": 1,
+            "pipeline_version": "evidence-authoring-v1",
+            "sources": {"../patient.pdf": {"sha256": "sha256:" + "a" * 64, "units": ["domain:patient"]}},
+            "orphans": [],
+        })
+
+
+def test_manifest_rejects_a_unit_declared_by_two_sources():
+    with pytest.raises(ValidationError, match="only one source"):
+        EvidenceManifest.model_validate({
+            "schema_version": 1,
+            "pipeline_version": "evidence-authoring-v1",
+            "sources": {
+                "source/domain/one.md": {
+                    "sha256": "sha256:" + "a" * 64,
+                    "units": ["evidence:shared"],
+                },
+                "source/domain/two.md": {
+                    "sha256": "sha256:" + "b" * 64,
+                    "units": ["evidence:shared"],
+                },
+            },
+            "orphans": [],
+        })
+
+
+def test_workspace_validation_rejects_duplicate_curated_ids_and_changed_sources(tmp_path):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    evidence = _evidence(source_text)
+    _write_workspace(tmp_path, evidence, source_text)
+    duplicate_path = tmp_path / "evidence" / "curated" / "domain" / "duplicate.md"
+    duplicate_path.write_text(dump_curated_markdown(evidence), encoding="utf-8")
+    (tmp_path / "evidence" / "source" / "domain" / "patient.md").write_text(
+        "Il testo sorgente è cambiato.", encoding="utf-8",
+    )
+
+    report = validate_workspace_evidence(tmp_path)
+
+    assert report.publishable is False
+    assert {finding.code for finding in report.findings} == {
+        "duplicate_evidence_id",
+        "source_hash_mismatch",
+        "supporting_excerpt_missing",
+    }
+
+
+def test_workspace_validation_reports_unreadable_or_oversized_sources(tmp_path, monkeypatch):
+    source_text = "I pazienti sotto i 18 anni sono pediatrici."
+    _write_workspace(tmp_path, _evidence(source_text), source_text)
+    source_path = tmp_path / "evidence" / "source" / "domain" / "patient.md"
+    source_path.write_bytes(b"\xff")
+
+    unreadable = validate_workspace_evidence(tmp_path)
+
+    assert [finding.code for finding in unreadable.findings] == ["source_unreadable"]
+
+    source_path.write_text(source_text, encoding="utf-8")
+    monkeypatch.setattr("tht.evidence.authoring.MAX_AUTHORING_FILE_BYTES", 1)
+
+    oversized = validate_workspace_evidence(tmp_path)
+
+    assert [finding.code for finding in oversized.findings] == ["source_oversized"]