From 0caa7479146f966d4c5e75d2e5f8d21ea566ef07 Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 25 Aug 2026 01:59:27 +0200 Subject: [PATCH] fix(evidence): validate curated runtime corpus --- PROJECT_STATE.md | 7 ++- backend/src/workspaces/runtime-renderer.ts | 5 +- backend/src/workspaces/schema.ts | 4 +- .../test/workspace-runtime-renderer.test.ts | 1 + backend/test/workspaces-schema.test.ts | 20 +++++++ docs/architecture/overview.md | 2 +- docs/contracts/workspace-evidence-v3.md | 10 ++-- docs/contracts/workspace-preprocessing-cli.md | 2 +- harness/tests/test_preprocess_cli.py | 55 ++++++++++++++++++- harness/tht/cli/preprocess_cmd.py | 13 +++++ harness/tht/config.py | 2 + 11 files changed, 107 insertions(+), 14 deletions(-) diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 61f0a55b..db6359b9 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -228,9 +228,10 @@ records remain revision-scoped; corpus ACTIVE is revision-qualified. HTTP/S3 Evidence is unchanged. - **Curated-only runtime contract (Evidence schema v2):** the new authoring layout preserves the complete commit-addressed `evidence/` tree (`source/`, `curated/`, manifest and evaluation files), - while the rendered filesystem acquisition default is only `curated/**/*.md`. Version 2 rejects - source or mixed source/curated runtime patterns; legacy Evidence version 1 retains its explicit - safe-pattern compatibility. The curator validates before merge and the runtime validates the + while the rendered filesystem acquisition pattern is exactly `curated/**/*.md`. Version 2 rejects + every other pattern, including source, mixed source/curated, broad curated, and non-Markdown + patterns; legacy Evidence version 1 retains its explicit safe-pattern compatibility. The curator + validates before merge and the runtime validates the pinned curated corpus before indexing. The existing unnamed dense vector remains intact while Evidence may add `bm25`/`idf` additively; no runtime operation writes the authoring repository. - **Retention:** materialized roots live inside the commit-addressed snapshot directory, so they are diff --git a/backend/src/workspaces/runtime-renderer.ts b/backend/src/workspaces/runtime-renderer.ts index 23db59a1..4af55d8b 100644 --- a/backend/src/workspaces/runtime-renderer.ts +++ b/backend/src/workspaces/runtime-renderer.ts @@ -174,7 +174,10 @@ function renderEvidence( } return { - evidence: { sources: [renderedSource] }, + evidence: { + ...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}), + sources: [renderedSource], + }, vector: { max_chunk_chars: workspace.evidence.policy.max_chunk_chars, retain_published_generations: workspace.evidence.policy.retain_published_generations, diff --git a/backend/src/workspaces/schema.ts b/backend/src/workspaces/schema.ts index 9ba9d75d..66c5363c 100644 --- a/backend/src/workspaces/schema.ts +++ b/backend/src/workspaces/schema.ts @@ -341,11 +341,11 @@ const workspaceEvidenceSchema = z.object({ path: ["source", "patterns"], message: "schema-versioned filesystem Evidence patterns cannot span source and curated", }); - } else if (!patterns.every((pattern) => pattern === "curated" || pattern.startsWith("curated/"))) { + } else if (patterns.length !== 1 || patterns[0] !== "curated/**/*.md") { context.addIssue({ code: "custom", path: ["source", "patterns"], - message: "schema-versioned filesystem Evidence patterns must acquire curated documents only", + message: "schema-versioned filesystem Evidence patterns must be exactly curated/**/*.md", }); } }).transform((evidence) => ({ diff --git a/backend/test/workspace-runtime-renderer.test.ts b/backend/test/workspace-runtime-renderer.test.ts index 28b80145..d5d3c5ea 100644 --- a/backend/test/workspace-runtime-renderer.test.ts +++ b/backend/test/workspace-runtime-renderer.test.ts @@ -207,6 +207,7 @@ test("renders filesystem Evidence below the immutable revision content root with expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision); expect(rendered.evidence).toEqual({ + schema_version: 2, sources: [{ type: "filesystem", root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`, diff --git a/backend/test/workspaces-schema.test.ts b/backend/test/workspaces-schema.test.ts index 5f8f70ef..07243da6 100644 --- a/backend/test/workspaces-schema.test.ts +++ b/backend/test/workspaces-schema.test.ts @@ -442,6 +442,26 @@ test("rejects a schema-versioned Evidence layout that acquires source documents }, /curated/i); }); +test.each(["curated/**/*.yaml", "curated/**"])( + "rejects a schema-versioned Evidence layout that uses the non-canonical curated pattern %s", + (pattern) => { + expectSafeEvidenceError({ + ...withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + }), + evidence: { + schema_version: 2, + source: { + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: [pattern], + }, + }, + }, /curated\/\*\*\/\*\.md/i); + }, +); + test("keeps evidence optional on schema v3", () => { expect(validateWorkspaceDescriptor(validWorkspaceObject())).not.toHaveProperty("evidence"); }); diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 7af73289..bbb57813 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -51,7 +51,7 @@ Il frontend renderizza questi widget-descriptor (registro in `src/widgets/`); il Il repository del workspace è il confine di pubblicazione: il curatore prepara `evidence/source/`, revisa le unità in `evidence/curated/`, valida e fa merge. Per `evidence.schema_version: 2` il runtime materializza l'intero albero `evidence/` dal commit Git esatto, ma il renderer consegna al -preprocessing soltanto `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest +preprocessing esattamente `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest ed evaluation restano disponibili solo per tracciabilità. Il runtime non modifica, stagea, committa o pubblica il repository di authoring. diff --git a/docs/contracts/workspace-evidence-v3.md b/docs/contracts/workspace-evidence-v3.md index b2f38ec9..3bfa0f7d 100644 --- a/docs/contracts/workspace-evidence-v3.md +++ b/docs/contracts/workspace-evidence-v3.md @@ -13,11 +13,11 @@ compatibility, where an omitted filesystem pattern defaults to `patterns: ["**/* `max_bytes: 10485760`. `evidence.schema_version: 2` declares the source/curated authoring layout. Its omitted filesystem -pattern defaults to `patterns: ["curated/**/*.md"]`; every explicit v2 filesystem pattern must also -remain below `curated/`. A v2 descriptor that selects `source/`, or spans both `source/` and -`curated/`, is rejected. Explicit safe legacy filesystem patterns remain supported under Evidence -version 1. HTTP and S3 sources do not use filesystem layout patterns and retain their existing -contracts. +pattern defaults to `patterns: ["curated/**/*.md"]`; if declared, the only accepted v2 filesystem +pattern list is exactly `patterns: ["curated/**/*.md"]`. A v2 descriptor that selects `source/`, +spans both `source/` and `curated/`, uses a broader curated glob, or selects a non-Markdown file is +rejected. Explicit safe legacy filesystem patterns remain supported under Evidence version 1. HTTP +and S3 sources do not use filesystem layout patterns and retain their existing contracts. The v2 authoring tree is: diff --git a/docs/contracts/workspace-preprocessing-cli.md b/docs/contracts/workspace-preprocessing-cli.md index 14204a76..22662a80 100644 --- a/docs/contracts/workspace-preprocessing-cli.md +++ b/docs/contracts/workspace-preprocessing-cli.md @@ -106,7 +106,7 @@ tht --installation /thothii-installation.yaml workspace vector rebuild before any bytes are written and no partial root is published. - `preprocess evidence` and `preprocess run` operate directly on the materialized root; the temporary `evidence_materialization_required` stop is retired (the code remains only for - pre-P6 compatibility). For `evidence.schema_version: 2`, runtime acquisition receives only + pre-P6 compatibility). For `evidence.schema_version: 2`, runtime acquisition receives exactly `curated/**/*.md`; `source/` and support files remain in the materialized tree for traceability. HTTP/S3 Evidence is unchanged. - The curator validates Evidence before merge. Preprocessing validates the pinned curated corpus diff --git a/harness/tests/test_preprocess_cli.py b/harness/tests/test_preprocess_cli.py index 473eeb87..9ac21bea 100644 --- a/harness/tests/test_preprocess_cli.py +++ b/harness/tests/test_preprocess_cli.py @@ -2,14 +2,18 @@ import json from pathlib import Path from types import SimpleNamespace +import pytest from typer.testing import CliRunner from tht.cli import app -def _runtime_config(tmp_path: Path, name: str = "workspace.yaml") -> Path: +def _runtime_config( + tmp_path: Path, name: str = "workspace.yaml", evidence_schema_version: int | None = None, +) -> Path: path = tmp_path / name (tmp_path / "evidence").mkdir(exist_ok=True) + evidence_version = "" if evidence_schema_version is None else f"\n schema_version: {evidence_schema_version}" path.write_text( f""" runtime_identity: @@ -28,6 +32,7 @@ embeddings: model: qwen3-embedding:0.6b dim: 1024 evidence: +{evidence_version} sources: - type: filesystem root: {tmp_path / 'evidence'} @@ -40,6 +45,54 @@ roots: return path +def test_v2_invalid_materialized_corpus_stops_before_vector_store_construction(monkeypatch, tmp_path): + import tht.cli.preprocess_cmd as command + from tht.evidence import ValidationFinding, ValidationReport + + config = _runtime_config(tmp_path, evidence_schema_version=2) + vector_store_constructed = False + monkeypatch.setattr( + "tht.evidence.validate_workspace_evidence", + lambda root: ValidationReport((ValidationFinding( + "error", "manifest_missing", "manifest.yaml", "missing", + ),)), + ) + + def forbidden_vector_store(*args, **kwargs): + nonlocal vector_store_constructed + vector_store_constructed = True + raise AssertionError("invalid corpus must not reach vector upsert setup") + + monkeypatch.setattr("tht.adapters.factory.build_vector_store", forbidden_vector_store) + + with pytest.raises(RuntimeError, match="curated Evidence corpus is invalid"): + command.run_from_config(config) + + assert vector_store_constructed is False + + +def test_v2_valid_materialized_corpus_is_validated_before_preprocessing(monkeypatch, tmp_path): + import tht.cli.preprocess_cmd as command + from tht.evidence import ValidationReport + + config = _runtime_config(tmp_path, evidence_schema_version=2) + calls = [] + + class FakePipeline: + def run_as_job(self, **kwargs): + calls.append(("run", kwargs)) + return "completed" + + monkeypatch.setattr("tht.evidence.validate_workspace_evidence", lambda root: calls.append(("validate", root)) or ValidationReport(())) + monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: calls.append(("vector", require_write)) or object()) + monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda cfg: object()) + monkeypatch.setattr("tht.evidence.build_sources", lambda evidence: []) + monkeypatch.setattr("tht.evidence.build_preprocessing_pipeline", lambda **kwargs: FakePipeline()) + + assert command.run_from_config(config) == "completed" + assert calls[:2] == [("validate", tmp_path), ("vector", True)] + + def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path): import tht.cli.preprocess_cmd as command diff --git a/harness/tht/cli/preprocess_cmd.py b/harness/tht/cli/preprocess_cmd.py index a92c8aea..b6907648 100644 --- a/harness/tht/cli/preprocess_cmd.py +++ b/harness/tht/cli/preprocess_cmd.py @@ -75,6 +75,18 @@ def _candidate_evaluator(cfg, *, vector_store, embedder): return evaluate +def _validate_materialized_curated_corpus(cfg) -> None: + """Fail closed on a v2 pinned filesystem corpus before any vector write is possible.""" + if cfg.evidence is None or cfg.evidence.schema_version != 2: + return + if not any(source.type == "filesystem" for source in cfg.evidence.sources): + return + from tht.evidence import validate_workspace_evidence + + if not validate_workspace_evidence(_evaluation_workspace_root(cfg)).publishable: + raise RuntimeError("curated Evidence corpus is invalid") + + def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict: value = { **payload, @@ -158,6 +170,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = cfg = _load_config_or_exit(config) if cfg.embeddings is None: raise RuntimeError("embeddings are not configured") + _validate_materialized_curated_corpus(cfg) corpus_root = cfg.paths.artifacts.parent / "corpus" vector_store = build_vector_store(cfg, require_write=True) embedder = make_embedder(cfg.embeddings) diff --git a/harness/tht/config.py b/harness/tht/config.py index 93ac3d47..162b89f0 100644 --- a/harness/tht/config.py +++ b/harness/tht/config.py @@ -473,6 +473,8 @@ EvidenceSourceConfig = Annotated[ class EvidenceSourcesConfig(BaseModel): + # Version 2 is the materialized source/curated authoring layout. + schema_version: Literal[1, 2] = 1 # Legacy curated-tree configuration remains accepted during migration. source_root: Path | None = None # cartella curata a mano nell'ETL (relativa a source_root): unica fonte delle