fix(evidence): validate curated runtime corpus

This commit is contained in:
2026-08-25 01:59:27 +02:00
parent 9f0184388d
commit 0caa747914
11 changed files with 107 additions and 14 deletions
+4 -3
View File
@@ -228,9 +228,10 @@
records remain revision-scoped; corpus ACTIVE is revision-qualified. HTTP/S3 Evidence is unchanged.
- **Curated-only runtime contract (Evidence schema v2):** the new authoring layout preserves the
complete commit-addressed `evidence/` tree (`source/`, `curated/`, manifest and evaluation files),
while the rendered filesystem acquisition default is only `curated/**/*.md`. Version 2 rejects
source or mixed source/curated runtime patterns; legacy Evidence version 1 retains its explicit
safe-pattern compatibility. The curator validates before merge and the runtime validates the
while the rendered filesystem acquisition pattern is exactly `curated/**/*.md`. Version 2 rejects
every other pattern, including source, mixed source/curated, broad curated, and non-Markdown
patterns; legacy Evidence version 1 retains its explicit safe-pattern compatibility. The curator
validates before merge and the runtime validates the
pinned curated corpus before indexing. The existing unnamed dense vector remains intact while
Evidence may add `bm25`/`idf` additively; no runtime operation writes the authoring repository.
- **Retention:** materialized roots live inside the commit-addressed snapshot directory, so they are
+4 -1
View File
@@ -174,7 +174,10 @@ function renderEvidence(
}
return {
evidence: { sources: [renderedSource] },
evidence: {
...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}),
sources: [renderedSource],
},
vector: {
max_chunk_chars: workspace.evidence.policy.max_chunk_chars,
retain_published_generations: workspace.evidence.policy.retain_published_generations,
+2 -2
View File
@@ -341,11 +341,11 @@ const workspaceEvidenceSchema = z.object({
path: ["source", "patterns"],
message: "schema-versioned filesystem Evidence patterns cannot span source and curated",
});
} else if (!patterns.every((pattern) => pattern === "curated" || pattern.startsWith("curated/"))) {
} else if (patterns.length !== 1 || patterns[0] !== "curated/**/*.md") {
context.addIssue({
code: "custom",
path: ["source", "patterns"],
message: "schema-versioned filesystem Evidence patterns must acquire curated documents only",
message: "schema-versioned filesystem Evidence patterns must be exactly curated/**/*.md",
});
}
}).transform((evidence) => ({
@@ -207,6 +207,7 @@ test("renders filesystem Evidence below the immutable revision content root with
expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision);
expect(rendered.evidence).toEqual({
schema_version: 2,
sources: [{
type: "filesystem",
root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`,
+20
View File
@@ -442,6 +442,26 @@ test("rejects a schema-versioned Evidence layout that acquires source documents
}, /curated/i);
});
test.each(["curated/**/*.yaml", "curated/**"])(
"rejects a schema-versioned Evidence layout that uses the non-canonical curated pattern %s",
(pattern) => {
expectSafeEvidenceError({
...withEvidence({
type: "filesystem",
uri: "psd-clinical/evidence",
}),
evidence: {
schema_version: 2,
source: {
type: "filesystem",
uri: "psd-clinical/evidence",
patterns: [pattern],
},
},
}, /curated\/\*\*\/\*\.md/i);
},
);
test("keeps evidence optional on schema v3", () => {
expect(validateWorkspaceDescriptor(validWorkspaceObject())).not.toHaveProperty("evidence");
});
+1 -1
View File
@@ -51,7 +51,7 @@ Il frontend renderizza questi widget-descriptor (registro in `src/widgets/`); il
Il repository del workspace è il confine di pubblicazione: il curatore prepara `evidence/source/`,
revisa le unità in `evidence/curated/`, valida e fa merge. Per `evidence.schema_version: 2` il
runtime materializza l'intero albero `evidence/` dal commit Git esatto, ma il renderer consegna al
preprocessing soltanto `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest
preprocessing esattamente `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest
ed evaluation restano disponibili solo per tracciabilità. Il runtime non modifica, stagea, committa
o pubblica il repository di authoring.
+5 -5
View File
@@ -13,11 +13,11 @@ compatibility, where an omitted filesystem pattern defaults to `patterns: ["**/*
`max_bytes: 10485760`.
`evidence.schema_version: 2` declares the source/curated authoring layout. Its omitted filesystem
pattern defaults to `patterns: ["curated/**/*.md"]`; every explicit v2 filesystem pattern must also
remain below `curated/`. A v2 descriptor that selects `source/`, or spans both `source/` and
`curated/`, is rejected. Explicit safe legacy filesystem patterns remain supported under Evidence
version 1. HTTP and S3 sources do not use filesystem layout patterns and retain their existing
contracts.
pattern defaults to `patterns: ["curated/**/*.md"]`; if declared, the only accepted v2 filesystem
pattern list is exactly `patterns: ["curated/**/*.md"]`. A v2 descriptor that selects `source/`,
spans both `source/` and `curated/`, uses a broader curated glob, or selects a non-Markdown file is
rejected. Explicit safe legacy filesystem patterns remain supported under Evidence version 1. HTTP
and S3 sources do not use filesystem layout patterns and retain their existing contracts.
The v2 authoring tree is:
@@ -106,7 +106,7 @@ tht --installation <absolute>/thothii-installation.yaml workspace vector rebuild
before any bytes are written and no partial root is published.
- `preprocess evidence` and `preprocess run` operate directly on the materialized root; the
temporary `evidence_materialization_required` stop is retired (the code remains only for
pre-P6 compatibility). For `evidence.schema_version: 2`, runtime acquisition receives only
pre-P6 compatibility). For `evidence.schema_version: 2`, runtime acquisition receives exactly
`curated/**/*.md`; `source/` and support files remain in the materialized tree for traceability.
HTTP/S3 Evidence is unchanged.
- The curator validates Evidence before merge. Preprocessing validates the pinned curated corpus
+54 -1
View File
@@ -2,14 +2,18 @@ import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from typer.testing import CliRunner
from tht.cli import app
def _runtime_config(tmp_path: Path, name: str = "workspace.yaml") -> Path:
def _runtime_config(
tmp_path: Path, name: str = "workspace.yaml", evidence_schema_version: int | None = None,
) -> Path:
path = tmp_path / name
(tmp_path / "evidence").mkdir(exist_ok=True)
evidence_version = "" if evidence_schema_version is None else f"\n schema_version: {evidence_schema_version}"
path.write_text(
f"""
runtime_identity:
@@ -28,6 +32,7 @@ embeddings:
model: qwen3-embedding:0.6b
dim: 1024
evidence:
{evidence_version}
sources:
- type: filesystem
root: {tmp_path / 'evidence'}
@@ -40,6 +45,54 @@ roots:
return path
def test_v2_invalid_materialized_corpus_stops_before_vector_store_construction(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
from tht.evidence import ValidationFinding, ValidationReport
config = _runtime_config(tmp_path, evidence_schema_version=2)
vector_store_constructed = False
monkeypatch.setattr(
"tht.evidence.validate_workspace_evidence",
lambda root: ValidationReport((ValidationFinding(
"error", "manifest_missing", "manifest.yaml", "missing",
),)),
)
def forbidden_vector_store(*args, **kwargs):
nonlocal vector_store_constructed
vector_store_constructed = True
raise AssertionError("invalid corpus must not reach vector upsert setup")
monkeypatch.setattr("tht.adapters.factory.build_vector_store", forbidden_vector_store)
with pytest.raises(RuntimeError, match="curated Evidence corpus is invalid"):
command.run_from_config(config)
assert vector_store_constructed is False
def test_v2_valid_materialized_corpus_is_validated_before_preprocessing(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
from tht.evidence import ValidationReport
config = _runtime_config(tmp_path, evidence_schema_version=2)
calls = []
class FakePipeline:
def run_as_job(self, **kwargs):
calls.append(("run", kwargs))
return "completed"
monkeypatch.setattr("tht.evidence.validate_workspace_evidence", lambda root: calls.append(("validate", root)) or ValidationReport(()))
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: calls.append(("vector", require_write)) or object())
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda cfg: object())
monkeypatch.setattr("tht.evidence.build_sources", lambda evidence: [])
monkeypatch.setattr("tht.evidence.build_preprocessing_pipeline", lambda **kwargs: FakePipeline())
assert command.run_from_config(config) == "completed"
assert calls[:2] == [("validate", tmp_path), ("vector", True)]
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
+13
View File
@@ -75,6 +75,18 @@ def _candidate_evaluator(cfg, *, vector_store, embedder):
return evaluate
def _validate_materialized_curated_corpus(cfg) -> None:
"""Fail closed on a v2 pinned filesystem corpus before any vector write is possible."""
if cfg.evidence is None or cfg.evidence.schema_version != 2:
return
if not any(source.type == "filesystem" for source in cfg.evidence.sources):
return
from tht.evidence import validate_workspace_evidence
if not validate_workspace_evidence(_evaluation_workspace_root(cfg)).publishable:
raise RuntimeError("curated Evidence corpus is invalid")
def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict:
value = {
**payload,
@@ -158,6 +170,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
cfg = _load_config_or_exit(config)
if cfg.embeddings is None:
raise RuntimeError("embeddings are not configured")
_validate_materialized_curated_corpus(cfg)
corpus_root = cfg.paths.artifacts.parent / "corpus"
vector_store = build_vector_store(cfg, require_write=True)
embedder = make_embedder(cfg.embeddings)
+2
View File
@@ -473,6 +473,8 @@ EvidenceSourceConfig = Annotated[
class EvidenceSourcesConfig(BaseModel):
# Version 2 is the materialized source/curated authoring layout.
schema_version: Literal[1, 2] = 1
# Legacy curated-tree configuration remains accepted during migration.
source_root: Path | None = None
# cartella curata a mano nell'ETL (relativa a source_root): unica fonte delle