fix(evidence): validate curated runtime corpus
This commit is contained in:
+4
-3
@@ -228,9 +228,10 @@
|
||||
records remain revision-scoped; corpus ACTIVE is revision-qualified. HTTP/S3 Evidence is unchanged.
|
||||
- **Curated-only runtime contract (Evidence schema v2):** the new authoring layout preserves the
|
||||
complete commit-addressed `evidence/` tree (`source/`, `curated/`, manifest and evaluation files),
|
||||
while the rendered filesystem acquisition default is only `curated/**/*.md`. Version 2 rejects
|
||||
source or mixed source/curated runtime patterns; legacy Evidence version 1 retains its explicit
|
||||
safe-pattern compatibility. The curator validates before merge and the runtime validates the
|
||||
while the rendered filesystem acquisition pattern is exactly `curated/**/*.md`. Version 2 rejects
|
||||
every other pattern, including source, mixed source/curated, broad curated, and non-Markdown
|
||||
patterns; legacy Evidence version 1 retains its explicit safe-pattern compatibility. The curator
|
||||
validates before merge and the runtime validates the
|
||||
pinned curated corpus before indexing. The existing unnamed dense vector remains intact while
|
||||
Evidence may add `bm25`/`idf` additively; no runtime operation writes the authoring repository.
|
||||
- **Retention:** materialized roots live inside the commit-addressed snapshot directory, so they are
|
||||
|
||||
@@ -174,7 +174,10 @@ function renderEvidence(
|
||||
}
|
||||
|
||||
return {
|
||||
evidence: { sources: [renderedSource] },
|
||||
evidence: {
|
||||
...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}),
|
||||
sources: [renderedSource],
|
||||
},
|
||||
vector: {
|
||||
max_chunk_chars: workspace.evidence.policy.max_chunk_chars,
|
||||
retain_published_generations: workspace.evidence.policy.retain_published_generations,
|
||||
|
||||
@@ -341,11 +341,11 @@ const workspaceEvidenceSchema = z.object({
|
||||
path: ["source", "patterns"],
|
||||
message: "schema-versioned filesystem Evidence patterns cannot span source and curated",
|
||||
});
|
||||
} else if (!patterns.every((pattern) => pattern === "curated" || pattern.startsWith("curated/"))) {
|
||||
} else if (patterns.length !== 1 || patterns[0] !== "curated/**/*.md") {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["source", "patterns"],
|
||||
message: "schema-versioned filesystem Evidence patterns must acquire curated documents only",
|
||||
message: "schema-versioned filesystem Evidence patterns must be exactly curated/**/*.md",
|
||||
});
|
||||
}
|
||||
}).transform((evidence) => ({
|
||||
|
||||
@@ -207,6 +207,7 @@ test("renders filesystem Evidence below the immutable revision content root with
|
||||
|
||||
expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision);
|
||||
expect(rendered.evidence).toEqual({
|
||||
schema_version: 2,
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`,
|
||||
|
||||
@@ -442,6 +442,26 @@ test("rejects a schema-versioned Evidence layout that acquires source documents
|
||||
}, /curated/i);
|
||||
});
|
||||
|
||||
test.each(["curated/**/*.yaml", "curated/**"])(
|
||||
"rejects a schema-versioned Evidence layout that uses the non-canonical curated pattern %s",
|
||||
(pattern) => {
|
||||
expectSafeEvidenceError({
|
||||
...withEvidence({
|
||||
type: "filesystem",
|
||||
uri: "psd-clinical/evidence",
|
||||
}),
|
||||
evidence: {
|
||||
schema_version: 2,
|
||||
source: {
|
||||
type: "filesystem",
|
||||
uri: "psd-clinical/evidence",
|
||||
patterns: [pattern],
|
||||
},
|
||||
},
|
||||
}, /curated\/\*\*\/\*\.md/i);
|
||||
},
|
||||
);
|
||||
|
||||
test("keeps evidence optional on schema v3", () => {
|
||||
expect(validateWorkspaceDescriptor(validWorkspaceObject())).not.toHaveProperty("evidence");
|
||||
});
|
||||
|
||||
@@ -51,7 +51,7 @@ Il frontend renderizza questi widget-descriptor (registro in `src/widgets/`); il
|
||||
Il repository del workspace è il confine di pubblicazione: il curatore prepara `evidence/source/`,
|
||||
revisa le unità in `evidence/curated/`, valida e fa merge. Per `evidence.schema_version: 2` il
|
||||
runtime materializza l'intero albero `evidence/` dal commit Git esatto, ma il renderer consegna al
|
||||
preprocessing soltanto `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest
|
||||
preprocessing esattamente `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest
|
||||
ed evaluation restano disponibili solo per tracciabilità. Il runtime non modifica, stagea, committa
|
||||
o pubblica il repository di authoring.
|
||||
|
||||
|
||||
@@ -13,11 +13,11 @@ compatibility, where an omitted filesystem pattern defaults to `patterns: ["**/*
|
||||
`max_bytes: 10485760`.
|
||||
|
||||
`evidence.schema_version: 2` declares the source/curated authoring layout. Its omitted filesystem
|
||||
pattern defaults to `patterns: ["curated/**/*.md"]`; every explicit v2 filesystem pattern must also
|
||||
remain below `curated/`. A v2 descriptor that selects `source/`, or spans both `source/` and
|
||||
`curated/`, is rejected. Explicit safe legacy filesystem patterns remain supported under Evidence
|
||||
version 1. HTTP and S3 sources do not use filesystem layout patterns and retain their existing
|
||||
contracts.
|
||||
pattern defaults to `patterns: ["curated/**/*.md"]`; if declared, the only accepted v2 filesystem
|
||||
pattern list is exactly `patterns: ["curated/**/*.md"]`. A v2 descriptor that selects `source/`,
|
||||
spans both `source/` and `curated/`, uses a broader curated glob, or selects a non-Markdown file is
|
||||
rejected. Explicit safe legacy filesystem patterns remain supported under Evidence version 1. HTTP
|
||||
and S3 sources do not use filesystem layout patterns and retain their existing contracts.
|
||||
|
||||
The v2 authoring tree is:
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ tht --installation <absolute>/thothii-installation.yaml workspace vector rebuild
|
||||
before any bytes are written and no partial root is published.
|
||||
- `preprocess evidence` and `preprocess run` operate directly on the materialized root; the
|
||||
temporary `evidence_materialization_required` stop is retired (the code remains only for
|
||||
pre-P6 compatibility). For `evidence.schema_version: 2`, runtime acquisition receives only
|
||||
pre-P6 compatibility). For `evidence.schema_version: 2`, runtime acquisition receives exactly
|
||||
`curated/**/*.md`; `source/` and support files remain in the materialized tree for traceability.
|
||||
HTTP/S3 Evidence is unchanged.
|
||||
- The curator validates Evidence before merge. Preprocessing validates the pinned curated corpus
|
||||
|
||||
@@ -2,14 +2,18 @@ import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from tht.cli import app
|
||||
|
||||
|
||||
def _runtime_config(tmp_path: Path, name: str = "workspace.yaml") -> Path:
|
||||
def _runtime_config(
|
||||
tmp_path: Path, name: str = "workspace.yaml", evidence_schema_version: int | None = None,
|
||||
) -> Path:
|
||||
path = tmp_path / name
|
||||
(tmp_path / "evidence").mkdir(exist_ok=True)
|
||||
evidence_version = "" if evidence_schema_version is None else f"\n schema_version: {evidence_schema_version}"
|
||||
path.write_text(
|
||||
f"""
|
||||
runtime_identity:
|
||||
@@ -28,6 +32,7 @@ embeddings:
|
||||
model: qwen3-embedding:0.6b
|
||||
dim: 1024
|
||||
evidence:
|
||||
{evidence_version}
|
||||
sources:
|
||||
- type: filesystem
|
||||
root: {tmp_path / 'evidence'}
|
||||
@@ -40,6 +45,54 @@ roots:
|
||||
return path
|
||||
|
||||
|
||||
def test_v2_invalid_materialized_corpus_stops_before_vector_store_construction(monkeypatch, tmp_path):
|
||||
import tht.cli.preprocess_cmd as command
|
||||
from tht.evidence import ValidationFinding, ValidationReport
|
||||
|
||||
config = _runtime_config(tmp_path, evidence_schema_version=2)
|
||||
vector_store_constructed = False
|
||||
monkeypatch.setattr(
|
||||
"tht.evidence.validate_workspace_evidence",
|
||||
lambda root: ValidationReport((ValidationFinding(
|
||||
"error", "manifest_missing", "manifest.yaml", "missing",
|
||||
),)),
|
||||
)
|
||||
|
||||
def forbidden_vector_store(*args, **kwargs):
|
||||
nonlocal vector_store_constructed
|
||||
vector_store_constructed = True
|
||||
raise AssertionError("invalid corpus must not reach vector upsert setup")
|
||||
|
||||
monkeypatch.setattr("tht.adapters.factory.build_vector_store", forbidden_vector_store)
|
||||
|
||||
with pytest.raises(RuntimeError, match="curated Evidence corpus is invalid"):
|
||||
command.run_from_config(config)
|
||||
|
||||
assert vector_store_constructed is False
|
||||
|
||||
|
||||
def test_v2_valid_materialized_corpus_is_validated_before_preprocessing(monkeypatch, tmp_path):
|
||||
import tht.cli.preprocess_cmd as command
|
||||
from tht.evidence import ValidationReport
|
||||
|
||||
config = _runtime_config(tmp_path, evidence_schema_version=2)
|
||||
calls = []
|
||||
|
||||
class FakePipeline:
|
||||
def run_as_job(self, **kwargs):
|
||||
calls.append(("run", kwargs))
|
||||
return "completed"
|
||||
|
||||
monkeypatch.setattr("tht.evidence.validate_workspace_evidence", lambda root: calls.append(("validate", root)) or ValidationReport(()))
|
||||
monkeypatch.setattr("tht.adapters.factory.build_vector_store", lambda cfg, require_write: calls.append(("vector", require_write)) or object())
|
||||
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda cfg: object())
|
||||
monkeypatch.setattr("tht.evidence.build_sources", lambda evidence: [])
|
||||
monkeypatch.setattr("tht.evidence.build_preprocessing_pipeline", lambda **kwargs: FakePipeline())
|
||||
|
||||
assert command.run_from_config(config) == "completed"
|
||||
assert calls[:2] == [("validate", tmp_path), ("vector", True)]
|
||||
|
||||
|
||||
def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
|
||||
import tht.cli.preprocess_cmd as command
|
||||
|
||||
|
||||
@@ -75,6 +75,18 @@ def _candidate_evaluator(cfg, *, vector_store, embedder):
|
||||
return evaluate
|
||||
|
||||
|
||||
def _validate_materialized_curated_corpus(cfg) -> None:
|
||||
"""Fail closed on a v2 pinned filesystem corpus before any vector write is possible."""
|
||||
if cfg.evidence is None or cfg.evidence.schema_version != 2:
|
||||
return
|
||||
if not any(source.type == "filesystem" for source in cfg.evidence.sources):
|
||||
return
|
||||
from tht.evidence import validate_workspace_evidence
|
||||
|
||||
if not validate_workspace_evidence(_evaluation_workspace_root(cfg)).publishable:
|
||||
raise RuntimeError("curated Evidence corpus is invalid")
|
||||
|
||||
|
||||
def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict:
|
||||
value = {
|
||||
**payload,
|
||||
@@ -158,6 +170,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
||||
cfg = _load_config_or_exit(config)
|
||||
if cfg.embeddings is None:
|
||||
raise RuntimeError("embeddings are not configured")
|
||||
_validate_materialized_curated_corpus(cfg)
|
||||
corpus_root = cfg.paths.artifacts.parent / "corpus"
|
||||
vector_store = build_vector_store(cfg, require_write=True)
|
||||
embedder = make_embedder(cfg.embeddings)
|
||||
|
||||
@@ -473,6 +473,8 @@ EvidenceSourceConfig = Annotated[
|
||||
|
||||
|
||||
class EvidenceSourcesConfig(BaseModel):
|
||||
# Version 2 is the materialized source/curated authoring layout.
|
||||
schema_version: Literal[1, 2] = 1
|
||||
# Legacy curated-tree configuration remains accepted during migration.
|
||||
source_root: Path | None = None
|
||||
# cartella curata a mano nell'ETL (relativa a source_root): unica fonte delle
|
||||
|
||||
Reference in New Issue
Block a user