333 lines
10 KiB
Python
333 lines
10 KiB
Python
"""Typed, reviewable Evidence units stored in the workspace repository."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Literal
|
|
|
|
import sqlglot
|
|
import yaml
|
|
from pydantic import AnyHttpUrl, BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
from sqlglot import exp
|
|
|
|
from tht.evidence.contracts import validate_canonical_uri
|
|
|
|
|
|
class StrictModel(BaseModel):
|
|
"""Reject undeclared fields in the repository's canonical format."""
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
EvidenceKind = Literal[
|
|
"glossary",
|
|
"domain",
|
|
"enum",
|
|
"example",
|
|
"mapping",
|
|
"normalization",
|
|
"formula",
|
|
"reference",
|
|
]
|
|
EvidencePurpose = Literal[
|
|
"disambiguation",
|
|
"rewriting",
|
|
"schema_linking",
|
|
"sql_generation",
|
|
]
|
|
_IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_$]*"
|
|
_TABLE_IDENTIFIER = re.compile(rf"^{_IDENTIFIER}\.{_IDENTIFIER}$")
|
|
_COLUMN_IDENTIFIER = re.compile(rf"^{_IDENTIFIER}\.{_IDENTIFIER}\.{_IDENTIFIER}$")
|
|
MAX_CURATED_FILE_BYTES = 10 * 1024 * 1024
|
|
|
|
|
|
class EvidenceScope(StrictModel):
|
|
concepts: tuple[str, ...] = ()
|
|
tables: tuple[str, ...] = ()
|
|
columns: tuple[str, ...] = ()
|
|
|
|
@field_validator("tables")
|
|
@classmethod
|
|
def _validate_tables(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
|
return _validate_identifiers(value, _TABLE_IDENTIFIER, "tables")
|
|
|
|
@field_validator("columns")
|
|
@classmethod
|
|
def _validate_columns(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
|
return _validate_identifiers(value, _COLUMN_IDENTIFIER, "columns")
|
|
|
|
|
|
class EvidenceProvenance(StrictModel):
|
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
|
|
source_file: str
|
|
source_sha256: str
|
|
supporting_excerpts: tuple[str, ...]
|
|
|
|
@field_validator("source_file")
|
|
@classmethod
|
|
def _validate_source_file(cls, value: str) -> str:
|
|
return validate_source_file(value)
|
|
|
|
@field_validator("source_sha256")
|
|
@classmethod
|
|
def _validate_sha256(cls, value: str) -> str:
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", value):
|
|
raise ValueError("source_sha256 must be a sha256 digest")
|
|
return value
|
|
|
|
@field_validator("supporting_excerpts")
|
|
@classmethod
|
|
def _validate_excerpts(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
|
if not 1 <= len(value) <= 5:
|
|
raise ValueError("supporting_excerpts must contain one to five items")
|
|
if any(not excerpt.strip() or len(excerpt) > 1000 for excerpt in value):
|
|
raise ValueError("supporting excerpts must be nonempty and at most 1000 characters")
|
|
return value
|
|
|
|
|
|
class ReviewItem(StrictModel):
|
|
code: str
|
|
message: str
|
|
field: str | None = None
|
|
|
|
|
|
class FormulaPayload(StrictModel):
|
|
concept: str
|
|
columns: tuple[str, ...]
|
|
sql: str
|
|
|
|
@field_validator("columns")
|
|
@classmethod
|
|
def _validate_columns(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
|
return _validate_identifiers(value, _COLUMN_IDENTIFIER, "columns")
|
|
|
|
@field_validator("sql")
|
|
@classmethod
|
|
def _validate_expression(cls, value: str) -> str:
|
|
try:
|
|
statements = [statement for statement in sqlglot.parse(value, read="postgres") if statement]
|
|
except sqlglot.errors.ParseError as error:
|
|
raise ValueError("formula.sql must be valid PostgreSQL") from error
|
|
if len(statements) != 1:
|
|
raise ValueError("formula.sql must contain exactly one expression")
|
|
expression = statements[0]
|
|
if expression.find(exp.Select) is not None or expression.find(exp.With) is not None:
|
|
raise ValueError("formula.sql must not contain a query")
|
|
if any(
|
|
expression.find(statement_type) is not None
|
|
for statement_type in (
|
|
exp.Insert,
|
|
exp.Update,
|
|
exp.Delete,
|
|
exp.Create,
|
|
exp.Drop,
|
|
exp.Alter,
|
|
exp.Merge,
|
|
exp.TruncateTable,
|
|
exp.Grant,
|
|
exp.Revoke,
|
|
exp.Command,
|
|
exp.Values,
|
|
exp.Set,
|
|
exp.Table,
|
|
)
|
|
):
|
|
raise ValueError("formula.sql must not contain DDL or DML")
|
|
return value
|
|
|
|
|
|
class ReferencePayload(StrictModel):
|
|
url: AnyHttpUrl
|
|
label: str
|
|
description: str
|
|
|
|
@field_validator("url")
|
|
@classmethod
|
|
def _reject_credentials(cls, value: AnyHttpUrl) -> AnyHttpUrl:
|
|
validate_canonical_uri(str(value))
|
|
return value
|
|
|
|
|
|
class GlossaryPayload(StrictModel):
|
|
definition: str
|
|
synonyms: tuple[str, ...] = ()
|
|
variants: tuple[str, ...] = ()
|
|
|
|
|
|
class DomainPayload(StrictModel):
|
|
rule: str
|
|
|
|
|
|
class EnumPayload(StrictModel):
|
|
column: str
|
|
values: dict[str, str]
|
|
|
|
@field_validator("column")
|
|
@classmethod
|
|
def _validate_column(cls, value: str) -> str:
|
|
_validate_identifiers((value,), _COLUMN_IDENTIFIER, "column")
|
|
return value
|
|
|
|
|
|
class ExamplePayload(StrictModel):
|
|
question: str
|
|
interpretation: str
|
|
|
|
|
|
class MappingPayload(StrictModel):
|
|
concept: str
|
|
tables: tuple[str, ...]
|
|
columns: tuple[str, ...]
|
|
|
|
@field_validator("tables")
|
|
@classmethod
|
|
def _validate_tables(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
|
return _validate_identifiers(value, _TABLE_IDENTIFIER, "tables")
|
|
|
|
@field_validator("columns")
|
|
@classmethod
|
|
def _validate_columns(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
|
return _validate_identifiers(value, _COLUMN_IDENTIFIER, "columns")
|
|
|
|
|
|
class NormalizationPayload(StrictModel):
|
|
input: str
|
|
output: str
|
|
rule: str
|
|
|
|
|
|
EvidencePayload = (
|
|
GlossaryPayload
|
|
| DomainPayload
|
|
| EnumPayload
|
|
| ExamplePayload
|
|
| MappingPayload
|
|
| NormalizationPayload
|
|
| FormulaPayload
|
|
| ReferencePayload
|
|
)
|
|
|
|
|
|
_PAYLOAD_TYPE_BY_KIND = {
|
|
"glossary": GlossaryPayload,
|
|
"domain": DomainPayload,
|
|
"enum": EnumPayload,
|
|
"example": ExamplePayload,
|
|
"mapping": MappingPayload,
|
|
"normalization": NormalizationPayload,
|
|
"formula": FormulaPayload,
|
|
"reference": ReferencePayload,
|
|
}
|
|
_EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$")
|
|
|
|
|
|
class CuratedEvidence(StrictModel):
|
|
schema_version: Literal[1]
|
|
id: str
|
|
title: str
|
|
kind: EvidenceKind
|
|
purposes: tuple[EvidencePurpose, ...]
|
|
applies_to: EvidenceScope = Field(default_factory=EvidenceScope)
|
|
language: str
|
|
provenance: EvidenceProvenance
|
|
review_items: tuple[ReviewItem, ...] = ()
|
|
payload: EvidencePayload
|
|
|
|
@model_validator(mode="after")
|
|
def _validate_kind_payload(self) -> CuratedEvidence:
|
|
if not is_evidence_id(self.id):
|
|
raise ValueError("id must use the evidence:<slug> form")
|
|
expected = _PAYLOAD_TYPE_BY_KIND.get(self.kind)
|
|
if expected is not None and not isinstance(self.payload, expected):
|
|
raise ValueError(f"{self.kind} requires its typed payload")
|
|
return self
|
|
|
|
|
|
def parse_curated_markdown(text: str, *, path: Path | None = None) -> CuratedEvidence:
|
|
"""Parse the canonical frontmatter representation of one Curated Evidence unit."""
|
|
if not text.startswith("---\n"):
|
|
raise ValueError("curated evidence requires YAML frontmatter")
|
|
try:
|
|
_, frontmatter, body = text.split("---\n", 2)
|
|
except ValueError as error:
|
|
raise ValueError("curated evidence frontmatter is malformed") from error
|
|
raw = yaml.safe_load(frontmatter)
|
|
try:
|
|
data = dict(raw)
|
|
except (TypeError, ValueError) as error:
|
|
raise ValueError("curated evidence frontmatter must be a mapping") from error
|
|
if body.strip():
|
|
raise ValueError("curated evidence must not contain an ignored body")
|
|
kind = data.get("kind")
|
|
if "payload" not in data and kind in _PAYLOAD_TYPE_BY_KIND:
|
|
data["payload"] = data.pop(kind, None)
|
|
evidence = CuratedEvidence.model_validate(data)
|
|
if path is not None:
|
|
_validate_kind_directory(path, evidence.kind)
|
|
return evidence
|
|
|
|
|
|
def dump_curated_markdown(value: CuratedEvidence) -> str:
|
|
"""Render canonical frontmatter with a human-readable kind-specific payload key."""
|
|
data = value.model_dump(mode="json", exclude={"payload"})
|
|
data[value.kind] = value.payload.model_dump(mode="json")
|
|
frontmatter = yaml.safe_dump(data, allow_unicode=True, sort_keys=False)
|
|
return f"---\n{frontmatter}---\n"
|
|
|
|
|
|
def load_curated_tree(root: Path) -> list[CuratedEvidence]:
|
|
"""Load canonical Evidence units in stable path order from a curated root."""
|
|
if not root.is_dir():
|
|
return []
|
|
documents: list[CuratedEvidence] = []
|
|
for path in sorted(root.rglob("*.md")):
|
|
if path.name.upper().startswith("README"):
|
|
continue
|
|
if path.stat().st_size > MAX_CURATED_FILE_BYTES:
|
|
raise ValueError("curated evidence exceeds the size limit")
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except UnicodeDecodeError as error:
|
|
raise ValueError("curated evidence must be UTF-8") from error
|
|
documents.append(parse_curated_markdown(text, path=path))
|
|
return documents
|
|
|
|
|
|
def _validate_kind_directory(path: Path, kind: EvidenceKind) -> None:
|
|
parts = path.parts
|
|
try:
|
|
curated_index = parts.index("curated")
|
|
except ValueError:
|
|
return
|
|
if len(parts) <= curated_index + 1 or parts[curated_index + 1] != kind:
|
|
raise ValueError("curated evidence kind must match its directory")
|
|
|
|
|
|
def _validate_identifiers(
|
|
values: tuple[str, ...], pattern: re.Pattern[str], field: str,
|
|
) -> tuple[str, ...]:
|
|
if any(pattern.fullmatch(value) is None for value in values):
|
|
raise ValueError(f"{field} must use canonical schema identifiers")
|
|
return values
|
|
|
|
|
|
def validate_source_file(value: str) -> str:
|
|
"""Validate a repository-relative, credential-free Source Evidence path."""
|
|
path = PurePosixPath(value)
|
|
if (
|
|
path.is_absolute()
|
|
or ".." in path.parts
|
|
or not path.parts
|
|
or path.parts[0] != "source"
|
|
or not value.endswith((".md", ".txt", ".sql.md"))
|
|
):
|
|
raise ValueError("source_file must be a supported path below source/")
|
|
return value
|
|
|
|
|
|
def is_evidence_id(value: str) -> bool:
|
|
"""Whether a value uses the stable public Evidence identifier format."""
|
|
return _EVIDENCE_ID.fullmatch(value) is not None
|