From fc83d29b5836b4db173e0874689426ecf2da526f Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 25 Aug 2026 02:17:05 +0200 Subject: [PATCH] test(evidence): record restructuring acceptance --- PROJECT_STATE.md | 12 + docs/testing/evidence-restructuring-manual.md | 80 +++++++ .../evidence_authoring/poorly_structured.md | 15 ++ .../test_evidence_restructuring_fixture.py | 17 ++ scripts/evidence-restructuring-acceptance.sh | 218 ++++++++++++++++++ 5 files changed, 342 insertions(+) create mode 100644 docs/testing/evidence-restructuring-manual.md create mode 100644 harness/tests/fixtures/evidence_authoring/poorly_structured.md create mode 100644 harness/tests/test_evidence_restructuring_fixture.py create mode 100755 scripts/evidence-restructuring-acceptance.sh diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index db6359b9..eac3fcfe 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,5 +1,17 @@ # ThothII — Project State +## Evidence restructuring owner gate (#46) — automated acceptance observed; PSD migration pending (#47) (2026-08-25) + +- **Observed local automation:** `bash scripts/evidence-restructuring-acceptance.sh` completed its + isolated fake-restructurer probe and the selected Evidence/L0 contract suite: **222 passed, 1 + known pytest deprecation warning**. The probe used only a `mktemp` workspace, confirmed the + ThothII worktree status was unchanged, and did not read or write an external PSD authoring path. +- **Scope boundary:** the realistic malformed source fixture, hermetic runner, and + `docs/testing/evidence-restructuring-manual.md` are ThothII-only artifacts. The real Pi call, + PSD source inventory, migration, Git review, vector before/after counts, and human walkthrough + remain **PENDING in issue #47** until the owner authorizes the migration window and exact target + branch. No PSD migration is represented by this entry. + ## Modular workflow refactor candidate — live (2026-08-24) - **Isolation:** worktree `.worktrees/refactoring-modulare-contract-baseline`, branch diff --git a/docs/testing/evidence-restructuring-manual.md b/docs/testing/evidence-restructuring-manual.md new file mode 100644 index 00000000..d0e60e5d --- /dev/null +++ b/docs/testing/evidence-restructuring-manual.md @@ -0,0 +1,80 @@ +# Evidence restructuring: owner migration gate + +Status: **PENDING OWNER AUTHORIZATION**. This guide records the manual work that must +occur only after the owner authorizes a migration window and exact PSD target branch. +The automated runner is hermetic: it uses a fake restructurer and temporary inputs; it +does not inspect, write, stage, or migrate the PSD authoring repository. + +## Recorded automated boundary + +Run from ThothII: + +```bash +bash scripts/evidence-restructuring-acceptance.sh +``` + +It proves the local contracts with an intentionally badly structured fixture: typed +splitting, review-item blocking, one-source membership, Git-visible proposals and +recoverability, no-op reruns, dirty-state refusal, pipeline-version refusal plus full +`--upgrade`, and orphan blocking. It also runs the hermetic authoring, canonical-kind, +formula, chunking, candidate-evaluation, hybrid-query/fail-closed, and pinned-Qdrant +L0 suites. The runner supplies only a `mktemp` workspace and asserts the ThothII +worktree is unchanged; consequently it performs no external PSD write. + +The real Pi invocation is deliberately not automated here. A reviewer must run it once +per changed source after the authorization gate and examine every proposed curated file. + +## Owner-gate package (issue #46) + +Before any PSD write, provide all of the following to the owner: + +- clean ThothII commit and the exact local-gate/acceptance output; +- proposed PSD branch name: `codex/evidence-restructuring-psd` (proposal only; no + external branch has been created); +- the exact, authorized-snapshot list of 36 source files to move; +- the pre-migration PSD commit and the rollback command + `git -C reset --hard `; +- before counts and representative IDs for `schema_table`, `schema_column`, `memory`, + and `solved_question`, plus dense-search samples for Schema and Memory. + +The 36-path inventory, PSD commit, and before counts are intentionally blank until the +owner authorizes the external repository inspection. Recording or executing them is +**issue #47**, not this issue. + +## Manual acceptance after authorization (issue #47) + +Record a separate PASS/FAIL and evidence for each item; never substitute an automated +test for a human Git review. + +1. **Authoring and Git review.** In the authorized PSD clone, move only the approved + 36 source paths to `evidence/source/`, run `tht evidence prepare`, verify exactly + one no-tool/no-session Pi call per changed source, inspect the Git diff, correct + every `review_item`, run `tht evidence validate`, and obtain the normal human Git + review. Confirm source renames/reclassifications retain IDs, semantic splits receive + new IDs, IDs use `evidence:`, and no orphan is deleted automatically. +2. **Additive BM25 schema upgrade.** Before preprocessing, run vector inspect and save + configuration, counts, and IDs. Run `workspace preprocess evidence`; verify the + unnamed dense vector remains and only `bm25` with IDF is added. Do not accept a + destructive rebuild, vector rename, or fallback engine. +3. **Schema and Memory non-regression.** Compare before/after counts and the saved + representative IDs for `schema_table`, `schema_column`, `memory`, and + `solved_question`; repeat dense Schema and Memory searches and attach the results. +4. **Preprocessing publication.** Confirm a validated corpus builds an inactive + candidate, evaluates that exact generation, and switches active generation only + after every evaluation query has an expected ID in the first ten fused hits. Attach + dense, BM25, and fused ranks for lexical, semantic, and mixed queries. +5. **Hybrid and formula retrieval.** Check dense and BM25 receive the identical NFC / + newline / outer-trim-only query text. Confirm Formula Evidence accepts a PostgreSQL + expression but rejects a full query, and retrieve one approved formula by its typed + Evidence path. +6. **Empty versus blocking unavailable.** Record one available empty retrieval and one + controlled Qdrant failure. The first may continue; the second must block the stage + without stale generation or purpose fallback. +7. **Complete session behavior.** Walk through `clarification`, `rewriting`, + `schema_linking`, `cte`, and `final_sql`; confirm independently persisted minimal + receipts. Confirm neither `memory` nor `synthesis` invokes Evidence search and that + session formula proposals remain unpublished. + +Write the reviewer identity, UTC time, commit IDs, command output locations, and one +final `manual acceptance: PASS` or `manual acceptance: FAIL` line when (and only when) +the authorized walkthrough is complete. diff --git a/harness/tests/fixtures/evidence_authoring/poorly_structured.md b/harness/tests/fixtures/evidence_authoring/poorly_structured.md new file mode 100644 index 00000000..573f75ba --- /dev/null +++ b/harness/tests/fixtures/evidence_authoring/poorly_structured.md @@ -0,0 +1,15 @@ +# Appunti clinici non strutturati + +La fascia pediatrica riguarda i pazienti con età minore di 18 anni; il testo non chiarisce se l'età debba essere calcolata alla data di ricovero o alla data odierna. + +Promemoria grezzo: + +- paziente pediatrico: meno di 18 anni +- paziente adulto: 18 anni o più +- controllare `clinical.patient.birth_date` + +Lo stato di dimissione usa i codici: D = dimesso, T = trasferito. + +Per il dettaglio clinico consultare https://example.test/linee-guida-dimissione . + +Formula candidata: `CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END`. diff --git a/harness/tests/test_evidence_restructuring_fixture.py b/harness/tests/test_evidence_restructuring_fixture.py new file mode 100644 index 00000000..ecaf85e8 --- /dev/null +++ b/harness/tests/test_evidence_restructuring_fixture.py @@ -0,0 +1,17 @@ +from pathlib import Path + + +def test_poorly_structured_authoring_fixture_exercises_the_restructuring_boundary(): + """Removing any rough source shape would weaken the hermetic acceptance probe.""" + fixture = ( + Path(__file__).parent / "fixtures" / "evidence_authoring" / "poorly_structured.md" + ) + + text = fixture.read_text(encoding="utf-8") + + assert "La fascia pediatrica" in text # prose + assert "- paziente" in text # rough list + assert "D = dimesso" in text # enum + assert "https://" in text # URL + assert "non chiarisce" in text # ambiguity + assert "CASE WHEN" in text # PostgreSQL expression candidate diff --git a/scripts/evidence-restructuring-acceptance.sh b/scripts/evidence-restructuring-acceptance.sh new file mode 100755 index 00000000..7b4d1438 --- /dev/null +++ b/scripts/evidence-restructuring-acceptance.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" +python="$repo_root/harness/.venv/bin/python" +fixture="$repo_root/harness/tests/fixtures/evidence_authoring/poorly_structured.md" + +[[ -x "$python" ]] || { printf 'missing harness virtualenv: %s\n' "$python" >&2; exit 127; } +[[ -f "$fixture" ]] || { printf 'missing authoring fixture: %s\n' "$fixture" >&2; exit 2; } + +before_status="$(git -C "$repo_root" status --porcelain)" +temporary_root="$(mktemp -d "${TMPDIR:-/tmp}/thothii-evidence-restructuring.XXXXXXXX")" +trap 'rm -rf -- "$temporary_root"' EXIT HUP INT TERM + +# The authoring target is created under mktemp and passed directly to the Python API. +# This runner has no PSD workspace argument or configured external authoring path. +TASK12_WORKSPACE="$temporary_root/workspace" TASK12_FIXTURE="$fixture" \ + PYTHONPATH="$repo_root/harness" "$python" - <<'PY' +import os +import shutil +import subprocess +from pathlib import Path + +from tht.evidence import ( + EvidencePreparationError, + EvidenceRestructurer, + RestructureCandidate, + dump_manifest, + load_curated_tree, + load_manifest, + prepare_workspace_evidence, + validate_workspace_evidence, +) + + +workspace = Path(os.environ["TASK12_WORKSPACE"]).resolve() +fixture = Path(os.environ["TASK12_FIXTURE"]).resolve() +source_root = workspace / "evidence" / "source" +source_root.mkdir(parents=True) +fixture_target = source_root / "notes" / "poorly-structured.md" +fixture_target.parent.mkdir() +shutil.copyfile(fixture, fixture_target) +independent_target = source_root / "notes" / "independent.md" +independent_target.write_text("La codifica ADT è usata soltanto per il triage.\n", encoding="utf-8") + + +class FakeRestructurer(EvidenceRestructurer): + """Hermetic, deterministic substitute for the one-call-per-source Pi boundary.""" + + def __init__(self): + self.requests = [] + + def restructure(self, request): + self.requests.append(request) + existing = {unit.title: unit.id for unit in request.previous_units} + + def candidate(title, kind, purposes, excerpt, payload, review_items=()): + return RestructureCandidate.model_validate({ + "schema_version": 1, + "existing_id": existing.get(title), + "title": title, + "kind": kind, + "purposes": purposes, + "applies_to": { + "concepts": ["fascia pediatrica"], + "tables": ["clinical.patient"], + "columns": ["clinical.patient.birth_date"], + }, + "language": "it", + "supporting_excerpts": [excerpt], + "review_items": review_items, + "payload": payload, + }) + + if request.source_file.endswith("independent.md"): + return (candidate( + "Codifica ADT", "domain", ["schema_linking"], + "La codifica ADT è usata soltanto per il triage.", + {"rule": "Usare ADT solo per il triage."}, + ),) + return ( + candidate( + "Fascia pediatrica", "formula", ["sql_generation"], + "Formula candidata: `CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END`.", + { + "concept": "fascia pediatrica", + "columns": ["clinical.patient.birth_date"], + "sql": "CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END", + }, + ), + candidate( + "Stato dimissione", "enum", ["schema_linking"], + "Lo stato di dimissione usa i codici: D = dimesso, T = trasferito.", + {"column": "clinical.episode.discharge_status", "values": {"D": "dimesso", "T": "trasferito"}}, + ), + candidate( + "Linea guida dimissione", "reference", ["rewriting"], + "https://example.test/linee-guida-dimissione", + { + "url": "https://example.test/linee-guida-dimissione", + "label": "Linea guida dimissione", + "description": "Riferimento clinico per la dimissione.", + }, + ), + candidate( + "Regola età", "domain", ["disambiguation"], + "La fascia pediatrica riguarda i pazienti con età minore di 18 anni; il testo non chiarisce se l'età debba essere calcolata alla data di ricovero o alla data odierna.", + {"rule": "La fascia pediatrica comprende i pazienti con meno di 18 anni."}, + [{ + "code": "ambiguous_source_statement", + "message": "La data di calcolo dell'età non è specificata.", + "field": "domain.rule", + }], + ), + ) + + +def run_git(*arguments): + return subprocess.run(["git", *arguments], cwd=workspace, check=True, capture_output=True, text=True) + + +fake = FakeRestructurer() +first = prepare_workspace_evidence(workspace, restructurer=fake, git_status=lambda _: ()) +assert first.model_calls == 2 and len(first.created) == 5 +documents = load_curated_tree(workspace / "evidence" / "curated") +assert {document.kind for document in documents} == {"domain", "enum", "formula", "reference"} +assert len({document.id for document in documents}) == 5 +assert {document.provenance.source_file for document in documents} == { + "source/notes/poorly-structured.md", "source/notes/independent.md", +} +manifest = load_manifest(workspace / "evidence" / "manifest.yaml") +assert not set(manifest.sources["source/notes/poorly-structured.md"].units).intersection( + manifest.sources["source/notes/independent.md"].units +) +assert "unresolved_review_item" in {finding.code for finding in validate_workspace_evidence(workspace).findings} + +run_git("init", "-q") +run_git("config", "user.email", "acceptance@example.test") +run_git("config", "user.name", "Acceptance") +run_git("add", "evidence") +run_git("commit", "-qm", "baseline human-reviewed curated evidence") +baseline_formula = run_git("show", "HEAD:evidence/curated/formula/fascia-pediatrica.md").stdout +fixture_target.write_text(fixture_target.read_text(encoding="utf-8") + "\nNota revisionata.\n", encoding="utf-8") +second = prepare_workspace_evidence(workspace, restructurer=fake, git_status=lambda _: ()) +assert second.model_calls == 1 and second.changed == ("source/notes/poorly-structured.md",) +changed_paths = set(run_git("diff", "--name-only").stdout.splitlines()) +assert "evidence/manifest.yaml" in changed_paths +assert "evidence/curated/formula/fascia-pediatrica.md" in changed_paths +assert run_git("show", "HEAD:evidence/curated/formula/fascia-pediatrica.md").stdout == baseline_formula + +snapshot = { + path.relative_to(workspace): path.read_bytes() + for path in (workspace / "evidence").rglob("*") if path.is_file() +} +no_op = prepare_workspace_evidence(workspace, restructurer=fake, git_status=lambda _: ()) +assert no_op.model_calls == 0 and no_op.changed == () +assert snapshot == { + path.relative_to(workspace): path.read_bytes() + for path in (workspace / "evidence").rglob("*") if path.is_file() +} + +try: + prepare_workspace_evidence( + workspace, restructurer=fake, git_status=lambda _: (" M evidence/curated/formula/fascia-pediatrica.md",), + ) +except EvidencePreparationError as failure: + assert failure.code == "authoring_worktree_dirty" +else: + raise AssertionError("dirty curated state was not refused") + +manifest_path = workspace / "evidence" / "manifest.yaml" +manifest_path.write_text( + manifest_path.read_text(encoding="utf-8").replace("evidence-authoring-v1", "evidence-authoring-v2"), + encoding="utf-8", +) +incompatible = manifest_path.read_bytes() +try: + prepare_workspace_evidence(workspace, restructurer=fake, git_status=lambda _: ()) +except EvidencePreparationError as failure: + assert failure.code == "pipeline_upgrade_required" +else: + raise AssertionError("incompatible pipeline version was not refused") +assert manifest_path.read_bytes() == incompatible +upgrade = prepare_workspace_evidence(workspace, restructurer=fake, git_status=lambda _: (), upgrade=True) +assert upgrade.model_calls == 2 and set(upgrade.changed) == { + "source/notes/independent.md", "source/notes/poorly-structured.md", +} + +manifest = load_manifest(manifest_path).model_copy(update={"orphans": ("evidence:fascia-pediatrica",)}) +manifest_path.write_text(dump_manifest(manifest), encoding="utf-8") +assert "orphaned_unit" in {finding.code for finding in validate_workspace_evidence(workspace).findings} + +print("PASS hermetic fake-restructurer: split, review, Git recovery/diff, no-op, dirty, upgrade, orphan") +print("PASS isolation: only a temporary workspace was supplied; no external PSD path was read or written") +PY + +( + cd "$repo_root/harness" + .venv/bin/pytest -q \ + tests/test_evidence_restructuring_fixture.py \ + tests/test_evidence_authoring.py \ + tests/test_evidence_canonical.py \ + tests/test_evidence_pi_restructurer.py \ + tests/test_evidence_evaluation.py \ + tests/test_evidence_formula_migration.py \ + tests/test_evidence_facade_contract.py \ + tests/test_corpus_chunk.py \ + tests/test_corpus_pipeline.py \ + tests/test_qdrant_vector_store.py \ + tests/l0/test_qdrant_bm25_inference.py +) + +after_status="$(git -C "$repo_root" status --porcelain)" +[[ "$before_status" == "$after_status" ]] || { + printf 'acceptance runner unexpectedly changed the ThothII worktree\n' >&2 + exit 1 +} +printf 'PASS evidence restructuring automated acceptance\n'