"""Typed, reviewable Evidence units stored in the workspace repository.""" from __future__ import annotations import re from pathlib import Path, PurePosixPath from typing import Literal import sqlglot import yaml from pydantic import AnyHttpUrl, BaseModel, ConfigDict, Field, field_validator, model_validator from sqlglot import exp from tht.evidence.contracts import validate_canonical_uri class StrictModel(BaseModel): """Reject undeclared fields in the repository's canonical format.""" model_config = ConfigDict(extra="forbid") EvidenceKind = Literal[ "glossary", "domain", "enum", "example", "mapping", "normalization", "formula", "reference", ] EvidencePurpose = Literal[ "disambiguation", "rewriting", "schema_linking", "sql_generation", ] _IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_$]*" _TABLE_IDENTIFIER = re.compile(rf"^{_IDENTIFIER}\.{_IDENTIFIER}$") _COLUMN_IDENTIFIER = re.compile(rf"^{_IDENTIFIER}\.{_IDENTIFIER}\.{_IDENTIFIER}$") MAX_CURATED_FILE_BYTES = 10 * 1024 * 1024 class EvidenceScope(StrictModel): concepts: tuple[str, ...] = () tables: tuple[str, ...] = () columns: tuple[str, ...] = () @field_validator("tables") @classmethod def _validate_tables(cls, value: tuple[str, ...]) -> tuple[str, ...]: return _validate_identifiers(value, _TABLE_IDENTIFIER, "tables") @field_validator("columns") @classmethod def _validate_columns(cls, value: tuple[str, ...]) -> tuple[str, ...]: return _validate_identifiers(value, _COLUMN_IDENTIFIER, "columns") class EvidenceProvenance(StrictModel): model_config = ConfigDict(extra="forbid", frozen=True) source_file: str source_sha256: str supporting_excerpts: tuple[str, ...] @field_validator("source_file") @classmethod def _validate_source_file(cls, value: str) -> str: return validate_source_file(value) @field_validator("source_sha256") @classmethod def _validate_sha256(cls, value: str) -> str: if not re.fullmatch(r"sha256:[0-9a-f]{64}", value): raise ValueError("source_sha256 must be a sha256 digest") return value @field_validator("supporting_excerpts") @classmethod def _validate_excerpts(cls, value: tuple[str, ...]) -> tuple[str, ...]: if not 1 <= len(value) <= 5: raise ValueError("supporting_excerpts must contain one to five items") if any(not excerpt.strip() or len(excerpt) > 1000 for excerpt in value): raise ValueError("supporting excerpts must be nonempty and at most 1000 characters") return value class ReviewItem(StrictModel): code: str message: str field: str | None = None class FormulaPayload(StrictModel): concept: str columns: tuple[str, ...] sql: str @field_validator("columns") @classmethod def _validate_columns(cls, value: tuple[str, ...]) -> tuple[str, ...]: return _validate_identifiers(value, _COLUMN_IDENTIFIER, "columns") @field_validator("sql") @classmethod def _validate_expression(cls, value: str) -> str: try: statements = [statement for statement in sqlglot.parse(value, read="postgres") if statement] except sqlglot.errors.ParseError as error: raise ValueError("formula.sql must be valid PostgreSQL") from error if len(statements) != 1: raise ValueError("formula.sql must contain exactly one expression") expression = statements[0] if expression.find(exp.Select) is not None or expression.find(exp.With) is not None: raise ValueError("formula.sql must not contain a query") if any( expression.find(statement_type) is not None for statement_type in ( exp.Insert, exp.Update, exp.Delete, exp.Create, exp.Drop, exp.Alter, exp.Merge, exp.TruncateTable, exp.Grant, exp.Revoke, exp.Command, exp.Values, exp.Set, exp.Table, ) ): raise ValueError("formula.sql must not contain DDL or DML") return value class ReferencePayload(StrictModel): url: AnyHttpUrl label: str description: str @field_validator("url") @classmethod def _reject_credentials(cls, value: AnyHttpUrl) -> AnyHttpUrl: validate_canonical_uri(str(value)) return value class GlossaryPayload(StrictModel): definition: str synonyms: tuple[str, ...] = () variants: tuple[str, ...] = () class DomainPayload(StrictModel): rule: str class EnumPayload(StrictModel): column: str values: dict[str, str] @field_validator("column") @classmethod def _validate_column(cls, value: str) -> str: _validate_identifiers((value,), _COLUMN_IDENTIFIER, "column") return value class ExamplePayload(StrictModel): question: str interpretation: str class MappingPayload(StrictModel): concept: str tables: tuple[str, ...] columns: tuple[str, ...] @field_validator("tables") @classmethod def _validate_tables(cls, value: tuple[str, ...]) -> tuple[str, ...]: return _validate_identifiers(value, _TABLE_IDENTIFIER, "tables") @field_validator("columns") @classmethod def _validate_columns(cls, value: tuple[str, ...]) -> tuple[str, ...]: return _validate_identifiers(value, _COLUMN_IDENTIFIER, "columns") class NormalizationPayload(StrictModel): input: str output: str rule: str EvidencePayload = ( GlossaryPayload | DomainPayload | EnumPayload | ExamplePayload | MappingPayload | NormalizationPayload | FormulaPayload | ReferencePayload ) _PAYLOAD_TYPE_BY_KIND = { "glossary": GlossaryPayload, "domain": DomainPayload, "enum": EnumPayload, "example": ExamplePayload, "mapping": MappingPayload, "normalization": NormalizationPayload, "formula": FormulaPayload, "reference": ReferencePayload, } _EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$") class CuratedEvidence(StrictModel): schema_version: Literal[1] id: str title: str kind: EvidenceKind purposes: tuple[EvidencePurpose, ...] applies_to: EvidenceScope = Field(default_factory=EvidenceScope) language: str provenance: EvidenceProvenance review_items: tuple[ReviewItem, ...] = () payload: EvidencePayload @model_validator(mode="after") def _validate_kind_payload(self) -> CuratedEvidence: if not is_evidence_id(self.id): raise ValueError("id must use the evidence: form") expected = _PAYLOAD_TYPE_BY_KIND.get(self.kind) if expected is not None and not isinstance(self.payload, expected): raise ValueError(f"{self.kind} requires its typed payload") return self def parse_curated_markdown(text: str, *, path: Path | None = None) -> CuratedEvidence: """Parse the canonical frontmatter representation of one Curated Evidence unit.""" if not text.startswith("---\n"): raise ValueError("curated evidence requires YAML frontmatter") try: _, frontmatter, body = text.split("---\n", 2) except ValueError as error: raise ValueError("curated evidence frontmatter is malformed") from error raw = yaml.safe_load(frontmatter) try: data = dict(raw) except (TypeError, ValueError) as error: raise ValueError("curated evidence frontmatter must be a mapping") from error if body.strip(): raise ValueError("curated evidence must not contain an ignored body") kind = data.get("kind") if "payload" not in data and kind in _PAYLOAD_TYPE_BY_KIND: data["payload"] = data.pop(kind, None) evidence = CuratedEvidence.model_validate(data) if path is not None: _validate_kind_directory(path, evidence.kind) return evidence def dump_curated_markdown(value: CuratedEvidence) -> str: """Render canonical frontmatter with a human-readable kind-specific payload key.""" data = value.model_dump(mode="json", exclude={"payload"}) data[value.kind] = value.payload.model_dump(mode="json") frontmatter = yaml.safe_dump(data, allow_unicode=True, sort_keys=False) return f"---\n{frontmatter}---\n" def load_curated_tree(root: Path) -> list[CuratedEvidence]: """Load canonical Evidence units in stable path order from a curated root.""" if not root.is_dir(): return [] documents: list[CuratedEvidence] = [] for path in sorted(root.rglob("*.md")): if path.name.upper().startswith("README"): continue if path.stat().st_size > MAX_CURATED_FILE_BYTES: raise ValueError("curated evidence exceeds the size limit") try: text = path.read_text(encoding="utf-8") except UnicodeDecodeError as error: raise ValueError("curated evidence must be UTF-8") from error documents.append(parse_curated_markdown(text, path=path)) return documents def _validate_kind_directory(path: Path, kind: EvidenceKind) -> None: parts = path.parts try: curated_index = parts.index("curated") except ValueError: return if len(parts) <= curated_index + 1 or parts[curated_index + 1] != kind: raise ValueError("curated evidence kind must match its directory") def _validate_identifiers( values: tuple[str, ...], pattern: re.Pattern[str], field: str, ) -> tuple[str, ...]: if any(pattern.fullmatch(value) is None for value in values): raise ValueError(f"{field} must use canonical schema identifiers") return values def validate_source_file(value: str) -> str: """Validate a repository-relative, credential-free Source Evidence path.""" path = PurePosixPath(value) if ( path.is_absolute() or ".." in path.parts or not path.parts or path.parts[0] != "source" or not value.endswith((".md", ".txt", ".sql.md")) ): raise ValueError("source_file must be a supported path below source/") return value def is_evidence_id(value: str) -> bool: """Whether a value uses the stable public Evidence identifier format.""" return _EVIDENCE_ID.fullmatch(value) is not None