mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(evidence): build semantic fragments from typed units

Browse Source
This commit is contained in:
marcopan
2026-08-24 21:33:48 +02:00
parent 8adc085746
commit 3420c57c8b
9 changed files with 642 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/evidence/corpus/models.py
+37
View File
@@ -8,6 +8,7 @@ from typing import Self
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
from tht.evidence.canonical import EVIDENCE_KINDS, EVIDENCE_PURPOSES
from tht.evidence.contracts import (
canonical_provenance_uri,
normalize_aware_datetime,
@@ -17,6 +18,11 @@ from tht.evidence.contracts import (
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
_EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$")
_EVIDENCE_METADATA_KEYS = frozenset({
"evidence_id", "evidence_kind", "purposes", "scope", "language", "provenance",
})
_CHUNK_METADATA_KEYS = frozenset({"chunk_policy", "document", "source_fingerprint", "title"})
def _validate_namespaced_id(value: str) -> str:
@@ -37,6 +43,36 @@ def _require_content_hash(content: str, content_hash: str) -> None:
raise ValueError("content_hash must match the exact canonical UTF-8 content")
def _validate_evidence_metadata(metadata: Mapping[str, JsonValue]) -> None:
if not (set(metadata) & _EVIDENCE_METADATA_KEYS):
return
keys = set(metadata)
missing = _EVIDENCE_METADATA_KEYS - keys
unknown = keys - _EVIDENCE_METADATA_KEYS - _CHUNK_METADATA_KEYS
if missing or unknown:
raise ValueError("typed Evidence metadata must use the exact allowlisted keys")
if not isinstance(metadata["evidence_id"], str) or not _EVIDENCE_ID.fullmatch(metadata["evidence_id"]):
raise ValueError("typed Evidence metadata must contain a valid evidence_id")
if metadata["evidence_kind"] not in EVIDENCE_KINDS:
raise ValueError("typed Evidence metadata must contain a valid evidence_kind")
purposes = metadata["purposes"]
if not isinstance(purposes, list) or not purposes or any(value not in EVIDENCE_PURPOSES for value in purposes):
raise ValueError("typed Evidence metadata must contain public purposes")
scope = metadata["scope"]
if not isinstance(scope, dict) or set(scope) != {"concepts", "tables", "columns"}:
raise ValueError("typed Evidence metadata must contain canonical scope")
if any(not isinstance(scope[name], list) or any(not isinstance(value, str) for value in scope[name])
for name in ("concepts", "tables", "columns")):
raise ValueError("typed Evidence metadata must contain canonical scope")
if not isinstance(metadata["language"], str) or not metadata["language"]:
raise ValueError("typed Evidence metadata must contain language")
provenance = metadata["provenance"]
if not isinstance(provenance, dict) or set(provenance) != {
"source_file", "source_sha256", "supporting_excerpts",
}:
raise ValueError("typed Evidence metadata must contain canonical provenance")
class _CanonicalValue(BaseModel):
model_config = ConfigDict(
frozen=True, extra="forbid", validate_default=True, revalidate_instances="always"
@@ -99,6 +135,7 @@ class CanonicalChunk(_WithMetadata):
@model_validator(mode="after")
def content_hash_matches(self) -> "CanonicalChunk":
_require_content_hash(self.content, self.content_hash)
_validate_evidence_metadata(self.metadata)
return self