refactor(evidence): remove legacy Python layout (#31)
This commit is contained in:
@@ -1,14 +1,4 @@
|
||||
"""Cohesive public entrypoint for Evidence domain capabilities.
|
||||
|
||||
The implementation is introduced beside the legacy module layout so production callers can
|
||||
migrate one path at a time. These functions deliberately preserve the existing objects, ordering,
|
||||
and exceptions; they do not define a cross-domain service protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
"""Cohesive public entrypoint for Evidence domain capabilities."""
|
||||
|
||||
from tht.evidence.acquisition import acquire, discover
|
||||
from tht.evidence.contracts import (
|
||||
@@ -22,145 +12,17 @@ from tht.evidence.contracts import (
|
||||
validate_namespaced_value,
|
||||
validate_safe_metadata,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tht.config import EvidenceSourcesConfig
|
||||
from tht.corpus.chunk import ChunkPolicy
|
||||
from tht.corpus.pipeline import CorpusPipeline
|
||||
from tht.corpus.store import CorpusStore
|
||||
from tht.decisions import DecisionRecord
|
||||
from tht.evidence.preprocessing import EvidenceEmbedder
|
||||
from tht.ports.vector import VectorStore
|
||||
from tht.search.evidence import ActiveEvidenceSearcher
|
||||
from tht.session.models import SchemaLinking
|
||||
|
||||
|
||||
def build_sources(evidence: "EvidenceSourcesConfig | None") -> list[EvidenceSource]:
|
||||
"""Build configured Evidence source adapters in the existing deterministic order."""
|
||||
from tht.evidence.sources import build_sources as build_configured_sources
|
||||
|
||||
return build_configured_sources(evidence)
|
||||
|
||||
|
||||
def build_preprocessing_pipeline(
|
||||
*,
|
||||
store: "CorpusStore",
|
||||
sources: list[EvidenceSource],
|
||||
embedder: "EvidenceEmbedder",
|
||||
vector_store: "VectorStore",
|
||||
embedding_model: str,
|
||||
embedding_dimensions: int,
|
||||
chunk_policy: "ChunkPolicy",
|
||||
pipeline_version: str,
|
||||
retain_published_generations: int = 3,
|
||||
workspace_id: str | None = None,
|
||||
) -> "CorpusPipeline":
|
||||
"""Construct the Evidence preprocessing use case from core-owned infrastructure."""
|
||||
from tht.evidence.preprocessing import build_preprocessing_pipeline as build_pipeline
|
||||
|
||||
return build_pipeline(
|
||||
store=store,
|
||||
sources=sources,
|
||||
embedder=embedder,
|
||||
vector_store=vector_store,
|
||||
embedding_model=embedding_model,
|
||||
embedding_dimensions=embedding_dimensions,
|
||||
chunk_policy=chunk_policy,
|
||||
pipeline_version=pipeline_version,
|
||||
retain_published_generations=retain_published_generations,
|
||||
workspace_id=workspace_id,
|
||||
)
|
||||
|
||||
|
||||
def active_searcher(
|
||||
cfg,
|
||||
delegate,
|
||||
*,
|
||||
workspace_id: str | None = None,
|
||||
) -> ActiveEvidenceSearcher:
|
||||
"""Bind vector search to the atomically ACTIVE Evidence corpus generation."""
|
||||
from tht.search.evidence import active_searcher as legacy_active_searcher
|
||||
|
||||
return legacy_active_searcher(cfg, delegate, workspace_id=workspace_id)
|
||||
|
||||
|
||||
def validate_corpus_workspace(cfg, workspace_id: str) -> None:
|
||||
"""Validate persisted Evidence corpus ownership before runtime retrieval setup."""
|
||||
from tht.search.evidence import validate_corpus_workspace as legacy_validate
|
||||
|
||||
legacy_validate(cfg, workspace_id)
|
||||
|
||||
|
||||
def build_retrieval_entries(results, *, excerpt_chars: int) -> list[dict]:
|
||||
"""Project ordered Evidence search hits into the existing retrieval-pack shape."""
|
||||
return [
|
||||
{
|
||||
"title": result.label,
|
||||
"status": result.status,
|
||||
"excerpt": result.content[:excerpt_chars],
|
||||
}
|
||||
for result in results
|
||||
]
|
||||
|
||||
|
||||
def resolve_citation(
|
||||
store: "CorpusStore",
|
||||
evidence_id: str,
|
||||
*,
|
||||
materialized_root: Path | None = None,
|
||||
) -> str:
|
||||
"""Resolve an Evidence identifier to its immutable ACTIVE materialization."""
|
||||
from tht.search.evidence import resolve_evidence_file
|
||||
|
||||
return resolve_evidence_file(
|
||||
store,
|
||||
evidence_id,
|
||||
materialized_root=materialized_root,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_session_citation(evidence_root: Path, evidence_id: str) -> str:
|
||||
# New deployments resolve only immutable materialized files from ACTIVE. Keep the
|
||||
# curated-tree fallback for sessions created before a canonical corpus exists.
|
||||
corpus_root = evidence_root.parent.parent / "corpus"
|
||||
if corpus_root.exists():
|
||||
from tht.corpus.store import CorpusStore
|
||||
|
||||
return resolve_citation(
|
||||
CorpusStore(corpus_root),
|
||||
evidence_id,
|
||||
materialized_root=evidence_root.parent / ".materialized-evidence",
|
||||
)
|
||||
for match in evidence_root.rglob(f"{evidence_id}.md"):
|
||||
return str(match)
|
||||
return ""
|
||||
|
||||
|
||||
def project_session(
|
||||
decisions: list["DecisionRecord"],
|
||||
linking: "SchemaLinking",
|
||||
evidence_root: Path,
|
||||
) -> list[dict]:
|
||||
"""Project cited and reviewed Evidence into the existing session artifact shape."""
|
||||
entries: dict[str, dict] = {}
|
||||
for candidate in linking.candidates:
|
||||
for evidence_id in candidate.evidence:
|
||||
entries.setdefault(evidence_id, {
|
||||
"id": evidence_id,
|
||||
"file": _resolve_session_citation(evidence_root, evidence_id),
|
||||
"esito": "usata",
|
||||
"decision_seq": candidate.decision_seq,
|
||||
})
|
||||
for decision in decisions:
|
||||
if decision.type not in ("evidence_accepted", "evidence_rejected"):
|
||||
continue
|
||||
entries[decision.subject] = {
|
||||
"id": decision.subject,
|
||||
"file": _resolve_session_citation(evidence_root, decision.subject),
|
||||
"esito": "accettata" if decision.type == "evidence_accepted" else "scartata",
|
||||
"decision_seq": decision.seq,
|
||||
}
|
||||
return list(entries.values())
|
||||
from tht.evidence.preprocessing import EvidenceEmbedder, build_preprocessing_pipeline
|
||||
from tht.evidence.search import (
|
||||
ActiveEvidenceSearcher,
|
||||
CorpusWorkspaceMismatchError,
|
||||
active_searcher,
|
||||
build_retrieval_entries,
|
||||
resolve_citation,
|
||||
validate_corpus_workspace,
|
||||
)
|
||||
from tht.evidence.session import project_session
|
||||
from tht.evidence.sources import build_sources
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -168,7 +30,10 @@ __all__ = [
|
||||
"EvidenceSource",
|
||||
"EvidenceSourceError",
|
||||
"EvidenceSourceErrorCategory",
|
||||
"EvidenceEmbedder",
|
||||
"SourceObject",
|
||||
"ActiveEvidenceSearcher",
|
||||
"CorpusWorkspaceMismatchError",
|
||||
"acquire",
|
||||
"active_searcher",
|
||||
"build_preprocessing_pipeline",
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Evidence-owned source adapter implementations."""
|
||||
|
||||
from tht.evidence.adapters.filesystem import FilesystemEvidenceSource
|
||||
from tht.evidence.adapters.http import HttpManifestEvidenceSource
|
||||
from tht.evidence.adapters.s3 import S3EvidenceSource
|
||||
|
||||
__all__ = ["FilesystemEvidenceSource", "HttpManifestEvidenceSource", "S3EvidenceSource"]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Evidence-owned, race-safe filesystem source."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import stat
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from tht.evidence.contracts import (
|
||||
AcquiredDocument,
|
||||
EvidenceSourceError,
|
||||
EvidenceSourceErrorCategory,
|
||||
SourceObject,
|
||||
)
|
||||
|
||||
|
||||
class FilesystemEvidenceSource:
|
||||
def __init__(
|
||||
self,
|
||||
root: Path | str,
|
||||
*,
|
||||
patterns: tuple[str, ...] | list[str] = ("**/*.md",),
|
||||
max_bytes: int = 10 * 1024 * 1024,
|
||||
) -> None:
|
||||
if max_bytes < 1:
|
||||
raise ValueError("max_bytes must be positive")
|
||||
if not patterns or any(
|
||||
not pattern or Path(pattern).is_absolute() or ".." in Path(pattern).parts
|
||||
for pattern in patterns
|
||||
):
|
||||
raise ValueError("at least one non-empty discovery pattern is required")
|
||||
try:
|
||||
self.root = Path(root).expanduser().resolve(strict=True)
|
||||
self._root_fd = os.open(
|
||||
self.root,
|
||||
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
)
|
||||
except OSError as error:
|
||||
raise ValueError("filesystem evidence root is unavailable") from error
|
||||
self.patterns = tuple(patterns)
|
||||
self.max_bytes = max_bytes
|
||||
|
||||
def __del__(self):
|
||||
root_fd = getattr(self, "_root_fd", None)
|
||||
if root_fd is not None:
|
||||
try:
|
||||
os.close(root_fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _safe_error(operation: str, *, transient: bool = False, **details):
|
||||
return EvidenceSourceError(
|
||||
"filesystem source operation failed",
|
||||
category=(
|
||||
EvidenceSourceErrorCategory.TRANSIENT
|
||||
if transient
|
||||
else EvidenceSourceErrorCategory.PERMANENT
|
||||
),
|
||||
details={"operation": operation, **details},
|
||||
)
|
||||
|
||||
def _open_read(self, relative: PurePosixPath) -> tuple[bytes, os.stat_result]:
|
||||
parts = relative.parts
|
||||
if not parts or any(part in {"", ".", ".."} for part in parts):
|
||||
raise self._safe_error("path_validation")
|
||||
directory_fd = os.dup(self._root_fd)
|
||||
file_fd = None
|
||||
try:
|
||||
for component in parts[:-1]:
|
||||
next_fd = os.open(
|
||||
component,
|
||||
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
dir_fd=directory_fd,
|
||||
)
|
||||
os.close(directory_fd)
|
||||
directory_fd = next_fd
|
||||
file_fd = os.open(
|
||||
parts[-1],
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
dir_fd=directory_fd,
|
||||
)
|
||||
file_stat = os.fstat(file_fd)
|
||||
if not stat.S_ISREG(file_stat.st_mode):
|
||||
raise self._safe_error("path_validation")
|
||||
if file_stat.st_size > self.max_bytes:
|
||||
raise self._safe_error("read", limit_bytes=self.max_bytes)
|
||||
content = bytearray()
|
||||
while len(content) <= self.max_bytes:
|
||||
chunk = os.read(file_fd, min(64 * 1024, self.max_bytes + 1 - len(content)))
|
||||
if not chunk:
|
||||
break
|
||||
content.extend(chunk)
|
||||
if len(content) > self.max_bytes:
|
||||
raise self._safe_error("read", limit_bytes=self.max_bytes)
|
||||
return bytes(content), file_stat
|
||||
except EvidenceSourceError:
|
||||
raise
|
||||
except OSError as error:
|
||||
raise self._safe_error("open") from error
|
||||
finally:
|
||||
if file_fd is not None:
|
||||
os.close(file_fd)
|
||||
os.close(directory_fd)
|
||||
|
||||
def _item(
|
||||
self, relative: PurePosixPath, content: bytes, file_stat: os.stat_result
|
||||
) -> SourceObject:
|
||||
relative_text = relative.as_posix()
|
||||
return SourceObject(
|
||||
source_id=f"filesystem:{hashlib.sha256(relative_text.encode()).hexdigest()}",
|
||||
uri=(self.root / relative_text).as_uri(),
|
||||
fingerprint=f"sha256:{hashlib.sha256(content).hexdigest()}",
|
||||
modified_at=datetime.fromtimestamp(file_stat.st_mtime, tz=UTC),
|
||||
metadata={"relative_path": relative_text},
|
||||
)
|
||||
|
||||
def discover(self):
|
||||
candidates = {
|
||||
path.relative_to(self.root).as_posix()
|
||||
for pattern in self.patterns
|
||||
for path in self.root.glob(pattern)
|
||||
}
|
||||
for relative_text in sorted(candidates):
|
||||
relative = PurePosixPath(relative_text)
|
||||
content, file_stat = self._open_read(relative)
|
||||
yield self._item(relative, content, file_stat)
|
||||
|
||||
def acquire(self, item: SourceObject) -> AcquiredDocument:
|
||||
parsed = urlsplit(item.uri)
|
||||
if parsed.scheme != "file" or parsed.netloc or parsed.query or parsed.fragment:
|
||||
raise self._safe_error("acquire")
|
||||
try:
|
||||
relative = Path(unquote(parsed.path)).relative_to(self.root)
|
||||
except ValueError as error:
|
||||
raise self._safe_error("acquire") from error
|
||||
pure_relative = PurePosixPath(relative.as_posix())
|
||||
content, file_stat = self._open_read(pure_relative)
|
||||
expected = self._item(pure_relative, content, file_stat)
|
||||
if item.source_id != expected.source_id or item.fingerprint != expected.fingerprint:
|
||||
raise self._safe_error("acquire")
|
||||
return AcquiredDocument(
|
||||
source=expected,
|
||||
content=content,
|
||||
media_type="text/markdown" if relative.suffix.lower() == ".md" else None,
|
||||
acquired_at=datetime.now(UTC),
|
||||
)
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Evidence-owned explicit-manifest HTTP source with SSRF-safe bounded acquisition."""
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import socket
|
||||
from collections import OrderedDict
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from tht.evidence.contracts import (
|
||||
AcquiredDocument,
|
||||
EvidenceSourceError,
|
||||
EvidenceSourceErrorCategory,
|
||||
SourceObject,
|
||||
canonical_provenance_uri,
|
||||
)
|
||||
|
||||
|
||||
class HttpManifestEvidenceSource:
|
||||
def __init__(
|
||||
self,
|
||||
urls: list[str] | tuple[str, ...],
|
||||
*,
|
||||
connect_timeout: float = 5,
|
||||
read_timeout: float = 30,
|
||||
max_bytes: int = 10 * 1024 * 1024,
|
||||
max_redirects: int = 5,
|
||||
allow_private_hosts: bool = False,
|
||||
max_cache_bytes: int = 64 * 1024 * 1024,
|
||||
) -> None:
|
||||
if not urls:
|
||||
raise ValueError("HTTP evidence manifest must contain at least one URL")
|
||||
if (
|
||||
connect_timeout <= 0
|
||||
or read_timeout <= 0
|
||||
or max_bytes < 1
|
||||
or max_redirects < 0
|
||||
or max_cache_bytes < 1
|
||||
):
|
||||
raise ValueError("HTTP evidence limits must be positive")
|
||||
self._transport_by_uri: dict[str, str] = {}
|
||||
for url in urls:
|
||||
self._validate_url_shape(url)
|
||||
provenance = canonical_provenance_uri(url)
|
||||
if provenance in self._transport_by_uri:
|
||||
raise ValueError("HTTP evidence manifest contains duplicate canonical provenance")
|
||||
self._transport_by_uri[provenance] = url
|
||||
self.connect_timeout = connect_timeout
|
||||
self.read_timeout = read_timeout
|
||||
self.max_bytes = max_bytes
|
||||
self.max_redirects = max_redirects
|
||||
self.allow_private_hosts = allow_private_hosts
|
||||
self.max_cache_bytes = max_cache_bytes
|
||||
self._session = requests.Session()
|
||||
self._session.trust_env = False
|
||||
self._cache: OrderedDict[str, AcquiredDocument] = OrderedDict()
|
||||
# provenance -> (exact final effective URL, ETag, Last-Modified)
|
||||
self._validators: dict[str, tuple[str, str | None, str | None]] = {}
|
||||
self._cache_bytes = 0
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"HttpManifestEvidenceSource(objects={len(self._transport_by_uri)})"
|
||||
|
||||
@staticmethod
|
||||
def _safe_error(operation: str, *, transient: bool = False, **details):
|
||||
return EvidenceSourceError(
|
||||
"HTTP source operation failed",
|
||||
category=(
|
||||
EvidenceSourceErrorCategory.TRANSIENT
|
||||
if transient
|
||||
else EvidenceSourceErrorCategory.PERMANENT
|
||||
),
|
||||
details={"operation": operation, **details},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_url_shape(url: str) -> None:
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("HTTP evidence URLs must use http or https")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValueError("HTTP evidence URLs must not contain userinfo credentials")
|
||||
|
||||
@staticmethod
|
||||
def _source_id(uri: str) -> str:
|
||||
return f"http:{hashlib.sha256(uri.encode()).hexdigest()}"
|
||||
|
||||
@staticmethod
|
||||
def _normalized_ip(value: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address:
|
||||
address = ipaddress.ip_address(value.split("%", 1)[0])
|
||||
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
|
||||
return address.ipv4_mapped
|
||||
return address
|
||||
|
||||
def _resolve_allowed(self, url: str) -> set[ipaddress.IPv4Address | ipaddress.IPv6Address]:
|
||||
try:
|
||||
self._validate_url_shape(url)
|
||||
except ValueError as error:
|
||||
raise self._safe_error("url_validation") from error
|
||||
if self.allow_private_hosts:
|
||||
return set()
|
||||
parsed = urlsplit(url)
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
try:
|
||||
rows = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
|
||||
addresses = {self._normalized_ip(row[4][0]) for row in rows}
|
||||
except (OSError, ValueError) as error:
|
||||
raise self._safe_error("resolution", transient=True) from error
|
||||
if not addresses:
|
||||
raise self._safe_error("resolution", transient=True)
|
||||
# Reject the entire answer set if any address is private/reserved. Choosing only a public
|
||||
# member would leave DNS ordering as a policy bypass.
|
||||
if any(not address.is_global for address in addresses):
|
||||
raise self._safe_error("network_policy")
|
||||
return addresses
|
||||
|
||||
def _verify_peer(
|
||||
self,
|
||||
response,
|
||||
allowed: set[ipaddress.IPv4Address | ipaddress.IPv6Address],
|
||||
) -> None:
|
||||
if self.allow_private_hosts:
|
||||
return
|
||||
try:
|
||||
connection = response.raw._connection
|
||||
peer = self._normalized_ip(connection.sock.getpeername()[0])
|
||||
except (AttributeError, OSError, TypeError, ValueError) as error:
|
||||
raise self._safe_error("peer_validation", transient=True) from error
|
||||
if not peer.is_global or peer not in allowed:
|
||||
raise self._safe_error("network_policy")
|
||||
|
||||
@staticmethod
|
||||
def _status_category(status: int) -> EvidenceSourceErrorCategory:
|
||||
if status in {408, 425, 429} or 500 <= status <= 599:
|
||||
return EvidenceSourceErrorCategory.TRANSIENT
|
||||
return EvidenceSourceErrorCategory.PERMANENT
|
||||
|
||||
def _conditional_headers(self, provenance: str, request_url: str) -> dict[str, str]:
|
||||
cached = self._cache.get(self._source_id(provenance))
|
||||
validators = self._validators.get(provenance)
|
||||
if cached is None or validators is None:
|
||||
return {}
|
||||
final_url, etag, last_modified = validators
|
||||
if request_url != final_url:
|
||||
return {}
|
||||
headers = {}
|
||||
if etag:
|
||||
headers["If-None-Match"] = etag
|
||||
if last_modified:
|
||||
headers["If-Modified-Since"] = last_modified
|
||||
return headers
|
||||
|
||||
def _remember(
|
||||
self,
|
||||
provenance: str,
|
||||
final_url: str,
|
||||
document: AcquiredDocument,
|
||||
validators: tuple[str | None, str | None],
|
||||
) -> None:
|
||||
source_id = document.source.source_id
|
||||
old = self._cache.pop(source_id, None)
|
||||
if old is not None:
|
||||
self._cache_bytes -= len(old.content)
|
||||
self._cache[source_id] = document
|
||||
self._cache_bytes += len(document.content)
|
||||
self._validators[provenance] = (final_url, *validators)
|
||||
while self._cache and self._cache_bytes > self.max_cache_bytes:
|
||||
evicted_id, evicted = self._cache.popitem(last=False)
|
||||
self._cache_bytes -= len(evicted.content)
|
||||
for uri in tuple(self._validators):
|
||||
if self._source_id(uri) == evicted_id:
|
||||
del self._validators[uri]
|
||||
|
||||
def _download(self, transport_url: str, provenance: str) -> AcquiredDocument:
|
||||
current = transport_url
|
||||
try:
|
||||
for redirect_count in range(self.max_redirects + 1):
|
||||
headers = self._conditional_headers(provenance, current)
|
||||
allowed = self._resolve_allowed(current)
|
||||
response = None
|
||||
try:
|
||||
response = self._session.get(
|
||||
current,
|
||||
headers=headers,
|
||||
stream=True,
|
||||
allow_redirects=False,
|
||||
timeout=(self.connect_timeout, self.read_timeout),
|
||||
)
|
||||
self._verify_peer(response, allowed)
|
||||
if response.is_redirect:
|
||||
if redirect_count == self.max_redirects:
|
||||
raise self._safe_error("redirect")
|
||||
destination = urljoin(current, response.headers.get("Location", ""))
|
||||
try:
|
||||
self._validate_url_shape(destination)
|
||||
except ValueError as error:
|
||||
raise self._safe_error("redirect") from error
|
||||
# The next iteration binds validators to the exact destination URL.
|
||||
current = destination
|
||||
continue
|
||||
if response.status_code == 304:
|
||||
cached = self._cache.get(self._source_id(provenance))
|
||||
binding = self._validators.get(provenance)
|
||||
if (
|
||||
cached is None
|
||||
or not headers
|
||||
or binding is None
|
||||
or binding[0] != current
|
||||
):
|
||||
raise self._safe_error("conditional_response")
|
||||
self._cache.move_to_end(cached.source.source_id)
|
||||
return cached
|
||||
if not 200 <= response.status_code <= 299:
|
||||
raise EvidenceSourceError(
|
||||
"HTTP status failure",
|
||||
category=self._status_category(response.status_code),
|
||||
details={"operation": "download", "status": response.status_code},
|
||||
)
|
||||
length = response.headers.get("Content-Length")
|
||||
if length is not None and int(length) > self.max_bytes:
|
||||
raise self._safe_error("download", limit_bytes=self.max_bytes)
|
||||
content = bytearray()
|
||||
for chunk in response.iter_content(
|
||||
chunk_size=min(64 * 1024, self.max_bytes + 1)
|
||||
):
|
||||
content.extend(chunk)
|
||||
if len(content) > self.max_bytes:
|
||||
raise self._safe_error("download", limit_bytes=self.max_bytes)
|
||||
etag = response.headers.get("ETag")
|
||||
last_modified = response.headers.get("Last-Modified")
|
||||
media_type = (
|
||||
response.headers.get("Content-Type", "").split(";", 1)[0] or None
|
||||
)
|
||||
break
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
except EvidenceSourceError:
|
||||
raise
|
||||
except (requests.Timeout, requests.ConnectionError, TimeoutError) as error:
|
||||
raise self._safe_error("download", transient=True) from error
|
||||
except requests.RequestException as error:
|
||||
raise self._safe_error("download", transient=True) from error
|
||||
except (OSError, ValueError) as error:
|
||||
raise self._safe_error("download") from error
|
||||
|
||||
modified_at = None
|
||||
if etag:
|
||||
fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}"
|
||||
elif last_modified:
|
||||
try:
|
||||
modified_at = parsedate_to_datetime(last_modified).astimezone(UTC)
|
||||
fingerprint = f"last-modified:{int(modified_at.timestamp())}"
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
fingerprint = f"sha256:{hashlib.sha256(content).hexdigest()}"
|
||||
else:
|
||||
fingerprint = f"sha256:{hashlib.sha256(content).hexdigest()}"
|
||||
item = SourceObject(
|
||||
source_id=self._source_id(provenance),
|
||||
uri=provenance,
|
||||
fingerprint=fingerprint,
|
||||
modified_at=modified_at,
|
||||
)
|
||||
document = AcquiredDocument(
|
||||
source=item,
|
||||
content=bytes(content),
|
||||
media_type=media_type,
|
||||
acquired_at=datetime.now(UTC),
|
||||
)
|
||||
self._remember(provenance, current, document, (etag, last_modified))
|
||||
return document
|
||||
|
||||
def discover(self):
|
||||
for provenance in sorted(self._transport_by_uri):
|
||||
yield self._download(self._transport_by_uri[provenance], provenance).source
|
||||
|
||||
def acquire(self, item: SourceObject) -> AcquiredDocument:
|
||||
transport = self._transport_by_uri.get(item.uri)
|
||||
if transport is None or item.source_id != self._source_id(item.uri):
|
||||
raise self._safe_error("acquire")
|
||||
document = self._download(transport, item.uri)
|
||||
if document.source.fingerprint != item.fingerprint:
|
||||
raise self._safe_error("acquire")
|
||||
return document
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Evidence-owned bounded S3-compatible source using the supported boto3 client."""
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
from tht.evidence.contracts import (
|
||||
AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject,
|
||||
)
|
||||
|
||||
|
||||
class S3EvidenceSource:
|
||||
def __init__(self, *, bucket: str, prefix: str = "", endpoint_url: str | None = None,
|
||||
region: str | None = None, access_key: str | None = None,
|
||||
secret_key: str | None = None, session_token: str | None = None,
|
||||
trusted_endpoint: bool = False,
|
||||
allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False,
|
||||
max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000,
|
||||
max_pages: int = 100, page_size: int = 1000, client=None) -> None:
|
||||
bucket_valid = re.fullmatch(r"(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)"
|
||||
r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", bucket)
|
||||
try:
|
||||
ipaddress.ip_address(bucket)
|
||||
bucket_is_ip = True
|
||||
except ValueError:
|
||||
bucket_is_ip = False
|
||||
if (not bucket_valid or bucket_is_ip
|
||||
or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size))):
|
||||
raise ValueError("S3 evidence limits and bucket must be non-empty and positive")
|
||||
if (prefix.startswith("/") or len(prefix.encode()) > 1024
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in prefix)):
|
||||
raise ValueError("S3 prefix is invalid")
|
||||
if endpoint_url:
|
||||
parsed = urlsplit(endpoint_url)
|
||||
if parsed.username or parsed.password:
|
||||
raise ValueError("S3 endpoint must not contain credentials")
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
raise ValueError("S3 endpoint scheme must be exactly https or explicitly allowed http")
|
||||
if parsed.scheme == "http" and not allow_insecure_endpoint:
|
||||
raise ValueError("S3 endpoint must use HTTPS unless explicitly allowed")
|
||||
if not parsed.hostname:
|
||||
raise ValueError("S3 endpoint must include a hostname")
|
||||
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
|
||||
raise ValueError("S3 custom endpoint must be an origin root without query/fragment")
|
||||
if not trusted_endpoint:
|
||||
raise ValueError("S3 custom endpoint requires explicit trusted_endpoint opt-in")
|
||||
try:
|
||||
literal = ipaddress.ip_address(parsed.hostname)
|
||||
except ValueError:
|
||||
literal = None
|
||||
if literal is not None and not literal.is_global and not allow_private_endpoint:
|
||||
raise ValueError("S3 private endpoint requires explicit opt-in")
|
||||
self.bucket, self.prefix = bucket, prefix
|
||||
self.max_bytes, self.max_objects = max_bytes, max_objects
|
||||
self.max_pages, self.page_size = max_pages, min(page_size, 1000)
|
||||
if client is None:
|
||||
try:
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
except ImportError as exc: # pragma: no cover - deployment optional dependency
|
||||
raise RuntimeError("Install tht[s3] to use S3 Evidence") from exc
|
||||
client = boto3.client("s3", endpoint_url=endpoint_url, region_name=region,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
aws_session_token=session_token, verify=True,
|
||||
config=BotoConfig(s3={"addressing_style": "path"}))
|
||||
self._client = client
|
||||
self._items: dict[str, tuple[SourceObject, str]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _error(operation: str, transient: bool = False):
|
||||
return EvidenceSourceError("S3 source operation failed",
|
||||
category=(EvidenceSourceErrorCategory.TRANSIENT if transient
|
||||
else EvidenceSourceErrorCategory.PERMANENT),
|
||||
details={"operation": operation})
|
||||
|
||||
def discover(self):
|
||||
count = pages = 0
|
||||
try:
|
||||
token = None
|
||||
for _ in range(self.max_pages):
|
||||
params = {"Bucket": self.bucket, "Prefix": self.prefix,
|
||||
"MaxKeys": self.page_size}
|
||||
if token is not None:
|
||||
params["ContinuationToken"] = token
|
||||
page = self._client.list_objects_v2(**params)
|
||||
pages += 1
|
||||
for row in page.get("Contents", []):
|
||||
count += 1
|
||||
if count > self.max_objects:
|
||||
raise self._error("object_limit")
|
||||
key, etag = row.get("Key"), row.get("ETag")
|
||||
if (not isinstance(key, str) or not key or not key.startswith(self.prefix)
|
||||
or len(key.encode()) > 1024
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in key)):
|
||||
raise self._error("invalid_key")
|
||||
if not isinstance(etag, str) or not etag or len(etag) > 1024:
|
||||
raise self._error("missing_validator")
|
||||
uri = f"s3://{self.bucket}/{quote(key, safe='/')}"
|
||||
fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}"
|
||||
source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest()
|
||||
modified = row.get("LastModified")
|
||||
if modified is not None:
|
||||
modified = modified.astimezone(UTC)
|
||||
item = SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint,
|
||||
modified_at=modified,
|
||||
metadata={"size": int(row.get("Size", 0))})
|
||||
self._items[source_id] = (item, etag)
|
||||
yield item
|
||||
if not page.get("IsTruncated"):
|
||||
return
|
||||
token = page.get("NextContinuationToken")
|
||||
if not isinstance(token, str) or not token:
|
||||
raise self._error("list_continuation")
|
||||
raise self._error("list_limit")
|
||||
except EvidenceSourceError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise self._error("list", transient=True) from exc
|
||||
|
||||
def acquire(self, item: SourceObject) -> AcquiredDocument:
|
||||
binding = self._items.get(item.source_id)
|
||||
if binding is None or item != binding[0]:
|
||||
raise self._error("acquire")
|
||||
discovered, etag = binding
|
||||
key = discovered.uri.split(f"s3://{self.bucket}/", 1)[1]
|
||||
from urllib.parse import unquote
|
||||
key = unquote(key)
|
||||
kwargs = {"Bucket": self.bucket, "Key": key}
|
||||
body = None
|
||||
try:
|
||||
response = self._client.get_object(**kwargs)
|
||||
body = response["Body"]
|
||||
if response.get("ETag") != etag:
|
||||
raise self._error("etag_changed")
|
||||
if int(response.get("ContentLength", 0)) > self.max_bytes:
|
||||
raise self._error("download_limit")
|
||||
content = body.read(self.max_bytes + 1)
|
||||
if len(content) > self.max_bytes:
|
||||
raise self._error("download_limit")
|
||||
return AcquiredDocument(source=item, content=content,
|
||||
media_type=response.get("ContentType"),
|
||||
acquired_at=datetime.now(UTC))
|
||||
except EvidenceSourceError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise self._error("download", transient=True) from exc
|
||||
finally:
|
||||
if body is not None:
|
||||
body.close()
|
||||
@@ -0,0 +1 @@
|
||||
"""Evidence-owned canonical, transport-independent corpus."""
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Evidence-owned deterministic chunking for canonical corpus documents."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChunkPolicy:
|
||||
version: str
|
||||
max_chars: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.version:
|
||||
raise ValueError("chunk policy version must not be empty")
|
||||
if self.max_chars <= 0:
|
||||
raise ValueError("max_chars must be greater than zero")
|
||||
|
||||
|
||||
def _hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _contents(content: str, maximum: int) -> list[str]:
|
||||
result: list[str] = []
|
||||
start = 0
|
||||
while start < len(content):
|
||||
end = min(start + maximum, len(content))
|
||||
if end < len(content):
|
||||
boundaries = list(re.finditer(r"\s+", content[start:end]))
|
||||
if boundaries:
|
||||
end = start + boundaries[-1].end()
|
||||
result.append(content[start:end])
|
||||
start = end
|
||||
return result
|
||||
|
||||
|
||||
def _policy_fingerprint(policy: ChunkPolicy) -> str:
|
||||
serialized = json.dumps(asdict(policy), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return f"sha256:{_hash(serialized)}"
|
||||
|
||||
|
||||
def chunk(document: CanonicalDocument, policy: ChunkPolicy) -> list[CanonicalChunk]:
|
||||
"""Split canonical text with stable character-count boundaries and identifiers."""
|
||||
chunks: list[CanonicalChunk] = []
|
||||
policy_fingerprint = _policy_fingerprint(policy)
|
||||
for ordinal, content in enumerate(_contents(document.content, policy.max_chars)):
|
||||
chunk_hash = f"sha256:{_hash(content)}"
|
||||
identifier = _hash(
|
||||
":".join(
|
||||
(
|
||||
document.document_id,
|
||||
document.content_hash,
|
||||
policy_fingerprint,
|
||||
str(ordinal),
|
||||
chunk_hash,
|
||||
)
|
||||
)
|
||||
)
|
||||
chunks.append(
|
||||
CanonicalChunk(
|
||||
chunk_id=f"chunk:{identifier}",
|
||||
document_id=document.document_id,
|
||||
ordinal=ordinal,
|
||||
content=content,
|
||||
content_hash=chunk_hash,
|
||||
source_uri=document.source_uri,
|
||||
pipeline_version=document.pipeline_version,
|
||||
metadata={
|
||||
"chunk_policy": {
|
||||
"version": policy.version,
|
||||
"max_chars": policy.max_chars,
|
||||
"fingerprint": policy_fingerprint,
|
||||
},
|
||||
"document": document.model_dump(mode="json")["metadata"],
|
||||
"source_fingerprint": document.source_fingerprint,
|
||||
"title": document.title,
|
||||
},
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Evidence-owned immutable records emitted by preprocessing."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Self
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
||||
|
||||
from tht.evidence.contracts import (
|
||||
canonical_provenance_uri,
|
||||
normalize_aware_datetime,
|
||||
validate_namespaced_value,
|
||||
validate_safe_metadata,
|
||||
)
|
||||
|
||||
|
||||
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
||||
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def _validate_namespaced_id(value: str) -> str:
|
||||
if not _NAMESPACED_ID.fullmatch(value):
|
||||
raise ValueError("identifier must be namespaced as '<kind>:<stable-value>'")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_hash(value: str) -> str:
|
||||
if not _SHA256.fullmatch(value):
|
||||
raise ValueError("content hash must be 'sha256:' followed by 64 lowercase hex digits")
|
||||
return value
|
||||
|
||||
|
||||
def _require_content_hash(content: str, content_hash: str) -> None:
|
||||
expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
|
||||
if content_hash != expected:
|
||||
raise ValueError("content_hash must match the exact canonical UTF-8 content")
|
||||
|
||||
|
||||
class _CanonicalValue(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
frozen=True, extra="forbid", validate_default=True, revalidate_instances="always"
|
||||
)
|
||||
|
||||
def model_copy(self, *, update: Mapping[str, object] | None = None, deep: bool = False) -> Self:
|
||||
"""Copy through full field and model validation, including manifest invariants."""
|
||||
data = self.model_dump(round_trip=True)
|
||||
if update:
|
||||
data.update(update)
|
||||
return type(self).model_validate(data)
|
||||
|
||||
|
||||
class _WithMetadata(_CanonicalValue):
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
_frozen_metadata = field_validator("metadata")(validate_safe_metadata)
|
||||
|
||||
|
||||
class CanonicalDocument(_WithMetadata):
|
||||
"""Normalized text whose hash covers the exact stored UTF-8 content bytes."""
|
||||
document_id: str
|
||||
source_id: str
|
||||
source_uri: str
|
||||
source_fingerprint: str = Field(min_length=1)
|
||||
content_hash: str
|
||||
title: str = ""
|
||||
content: str
|
||||
media_type: str = "text/plain"
|
||||
modified_at: datetime | None = None
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
_document_id = field_validator("document_id")(_validate_namespaced_id)
|
||||
_source_id = field_validator("source_id")(_validate_namespaced_id)
|
||||
_source_uri = field_validator("source_uri")(canonical_provenance_uri)
|
||||
_source_fingerprint = field_validator("source_fingerprint")(validate_namespaced_value)
|
||||
_content_hash = field_validator("content_hash")(_validate_hash)
|
||||
_modified_at = field_validator("modified_at")(normalize_aware_datetime)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content_hash_matches(self) -> "CanonicalDocument":
|
||||
_require_content_hash(self.content, self.content_hash)
|
||||
return self
|
||||
|
||||
|
||||
class CanonicalChunk(_WithMetadata):
|
||||
"""Chunk text whose hash covers the exact stored UTF-8 content bytes."""
|
||||
chunk_id: str
|
||||
document_id: str
|
||||
ordinal: int = Field(ge=0)
|
||||
content: str
|
||||
content_hash: str
|
||||
source_uri: str
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
_chunk_id = field_validator("chunk_id")(_validate_namespaced_id)
|
||||
_document_id = field_validator("document_id")(_validate_namespaced_id)
|
||||
_content_hash = field_validator("content_hash")(_validate_hash)
|
||||
_source_uri = field_validator("source_uri")(canonical_provenance_uri)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content_hash_matches(self) -> "CanonicalChunk":
|
||||
_require_content_hash(self.content, self.content_hash)
|
||||
return self
|
||||
|
||||
|
||||
class CorpusManifest(_WithMetadata):
|
||||
"""Description of one internally consistent publishable generation."""
|
||||
|
||||
schema_version: int = Field(default=1, ge=1)
|
||||
manifest_id: str | None = None
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
pipeline_version: str = Field(default="evidence-v1", min_length=1)
|
||||
embedding_model: str | None = None
|
||||
embedding_dimensions: int | None = Field(default=None, gt=0)
|
||||
vector_generation: str | None = None
|
||||
documents: tuple[CanonicalDocument, ...] = Field(default_factory=tuple)
|
||||
chunks: tuple[CanonicalChunk, ...] = Field(default_factory=tuple)
|
||||
|
||||
_manifest_id = field_validator("manifest_id")(
|
||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||
)
|
||||
_vector_generation = field_validator("vector_generation")(
|
||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||
)
|
||||
_created_at = field_validator("created_at")(normalize_aware_datetime)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_generation(self) -> "CorpusManifest":
|
||||
if (self.embedding_model is None) != (self.embedding_dimensions is None):
|
||||
raise ValueError("embedding_model and embedding_dimensions must be set together")
|
||||
if self.vector_generation is not None and self.embedding_model is None:
|
||||
raise ValueError("vector_generation requires embedding model and dimension compatibility")
|
||||
|
||||
document_ids = [document.document_id for document in self.documents]
|
||||
source_ids = [document.source_id for document in self.documents]
|
||||
chunk_ids = [chunk.chunk_id for chunk in self.chunks]
|
||||
self._require_unique("document_id", document_ids)
|
||||
self._require_unique("source_id", source_ids)
|
||||
self._require_unique("chunk_id", chunk_ids)
|
||||
|
||||
documents = {document.document_id: document for document in self.documents}
|
||||
ordinals: dict[str, list[int]] = {}
|
||||
for document in self.documents:
|
||||
if document.pipeline_version != self.pipeline_version:
|
||||
raise ValueError("document pipeline_version must match manifest pipeline_version")
|
||||
for chunk in self.chunks:
|
||||
document = documents.get(chunk.document_id)
|
||||
if document is None:
|
||||
raise ValueError(f"chunk references unknown document: {chunk.document_id}")
|
||||
if chunk.pipeline_version != self.pipeline_version:
|
||||
raise ValueError("chunk pipeline_version must match manifest pipeline_version")
|
||||
if chunk.source_uri != document.source_uri:
|
||||
raise ValueError("chunk source_uri must match its document provenance")
|
||||
ordinals.setdefault(chunk.document_id, []).append(chunk.ordinal)
|
||||
for document_id, values in ordinals.items():
|
||||
if sorted(values) != list(range(len(values))):
|
||||
raise ValueError(f"chunk ordinals must be unique and contiguous for {document_id}")
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _require_unique(field: str, values: list[str]) -> None:
|
||||
if len(values) != len(set(values)):
|
||||
raise ValueError(f"{field} values must be unique")
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Evidence-owned deterministic conversion of acquired bytes into canonical text."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Mapping
|
||||
|
||||
import yaml
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from yaml.events import AliasEvent
|
||||
from yaml.nodes import MappingNode
|
||||
|
||||
from tht.evidence.corpus.models import CanonicalDocument
|
||||
from tht.evidence.contracts import AcquiredDocument, canonical_provenance_uri
|
||||
|
||||
|
||||
MAX_DOCUMENT_BYTES = 10 * 1024 * 1024
|
||||
_CHARSET = re.compile(r"(?:^|;)\s*charset\s*=\s*[\"']?([^;\s\"']+)", re.IGNORECASE)
|
||||
_FRONTMATTER = re.compile(r"\A---\n(.*?)\n---(?:\n|\Z)", re.DOTALL)
|
||||
_JSON_OBJECT = TypeAdapter(dict[str, JsonValue])
|
||||
_MAX_FRONTMATTER_DEPTH = 20
|
||||
_MAX_FRONTMATTER_NODES = 1000
|
||||
|
||||
|
||||
class _FrontmatterLoader(yaml.SafeLoader):
|
||||
"""SafeLoader with bounded structure and no YAML graph features."""
|
||||
|
||||
def __init__(self, stream) -> None:
|
||||
super().__init__(stream)
|
||||
self._depth = 0
|
||||
self._nodes = 0
|
||||
|
||||
def compose_node(self, parent, index):
|
||||
event = self.peek_event()
|
||||
if isinstance(event, AliasEvent) or getattr(event, "anchor", None) is not None:
|
||||
raise yaml.constructor.ConstructorError(None, None, "aliases are not allowed")
|
||||
self._depth += 1
|
||||
self._nodes += 1
|
||||
if self._depth > _MAX_FRONTMATTER_DEPTH or self._nodes > _MAX_FRONTMATTER_NODES:
|
||||
raise yaml.constructor.ConstructorError(None, None, "frontmatter is too complex")
|
||||
try:
|
||||
return super().compose_node(parent, index)
|
||||
finally:
|
||||
self._depth -= 1
|
||||
|
||||
def construct_mapping(self, node, deep=False):
|
||||
if not isinstance(node, MappingNode):
|
||||
return super().construct_mapping(node, deep=deep)
|
||||
seen: set[object] = set()
|
||||
for key_node, _ in node.value:
|
||||
key = self.construct_object(key_node, deep=deep)
|
||||
try:
|
||||
duplicate = key in seen
|
||||
seen.add(key)
|
||||
except TypeError as error:
|
||||
raise yaml.constructor.ConstructorError(
|
||||
None, None, "mapping keys must be scalar"
|
||||
) from error
|
||||
if duplicate:
|
||||
raise yaml.constructor.ConstructorError(None, None, "duplicate mapping key")
|
||||
return super().construct_mapping(node, deep=deep)
|
||||
|
||||
|
||||
class PermanentNormalizationError(ValueError):
|
||||
"""A deterministic input failure which retrying cannot repair."""
|
||||
|
||||
def __init__(self, reason: str) -> None:
|
||||
super().__init__(f"document normalization failed: {reason}")
|
||||
self.reason = reason
|
||||
self.permanent = True
|
||||
|
||||
|
||||
def _sha256(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _decode(acquired: AcquiredDocument) -> str:
|
||||
if len(acquired.content) > MAX_DOCUMENT_BYTES:
|
||||
raise PermanentNormalizationError("oversized")
|
||||
|
||||
media_type = acquired.media_type or "text/plain"
|
||||
charset = _CHARSET.search(media_type)
|
||||
if charset and charset.group(1).lower().replace("_", "-") not in {
|
||||
"utf-8",
|
||||
"utf8",
|
||||
"us-ascii",
|
||||
"ascii",
|
||||
}:
|
||||
raise PermanentNormalizationError("unsupported_charset")
|
||||
try:
|
||||
return acquired.content.decode("utf-8-sig", errors="strict")
|
||||
except UnicodeDecodeError as error:
|
||||
raise PermanentNormalizationError("undecodable") from error
|
||||
|
||||
|
||||
def _frontmatter(text: str) -> tuple[dict[str, JsonValue], str]:
|
||||
match = _FRONTMATTER.match(text)
|
||||
if match is None:
|
||||
return {}, text
|
||||
try:
|
||||
loaded = yaml.load(match.group(1), Loader=_FrontmatterLoader)
|
||||
if loaded is None:
|
||||
loaded = {}
|
||||
if not isinstance(loaded, Mapping):
|
||||
raise TypeError("frontmatter is not a mapping")
|
||||
metadata = _JSON_OBJECT.validate_python(dict(loaded))
|
||||
except (TypeError, UnicodeError, ValidationError, yaml.YAMLError) as error:
|
||||
raise PermanentNormalizationError("invalid_frontmatter") from error
|
||||
return metadata, text[match.end() :]
|
||||
|
||||
|
||||
def normalize(acquired: AcquiredDocument, pipeline_version: str) -> CanonicalDocument:
|
||||
"""Normalize one transport result without I/O or implicit data loss."""
|
||||
if not pipeline_version:
|
||||
raise ValueError("pipeline_version must not be empty")
|
||||
|
||||
decoded = _decode(acquired)
|
||||
canonical = unicodedata.normalize("NFC", decoded.replace("\r\n", "\n").replace("\r", "\n"))
|
||||
frontmatter, content = _frontmatter(canonical)
|
||||
source_uri = canonical_provenance_uri(acquired.source.uri)
|
||||
identity = f"{acquired.source.source_id}\n{source_uri}"
|
||||
media_type = (acquired.media_type or "text/plain").split(";", 1)[0].strip().lower()
|
||||
metadata: dict[str, JsonValue] = {
|
||||
"source": acquired.source.model_dump(mode="json")["metadata"],
|
||||
"acquisition": acquired.model_dump(mode="json")["metadata"],
|
||||
}
|
||||
if frontmatter:
|
||||
metadata["frontmatter"] = frontmatter
|
||||
|
||||
try:
|
||||
return CanonicalDocument(
|
||||
document_id=f"doc:{_sha256(identity)}",
|
||||
source_id=acquired.source.source_id,
|
||||
source_uri=source_uri,
|
||||
source_fingerprint=acquired.source.fingerprint,
|
||||
content_hash=f"sha256:{_sha256(content)}",
|
||||
title=str(frontmatter.get("title", "")),
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
modified_at=acquired.source.modified_at,
|
||||
pipeline_version=pipeline_version,
|
||||
metadata=metadata,
|
||||
)
|
||||
except ValidationError as error:
|
||||
if frontmatter:
|
||||
raise PermanentNormalizationError("invalid_frontmatter") from error
|
||||
raise
|
||||
@@ -0,0 +1,811 @@
|
||||
"""Evidence-owned incremental preprocessing with generation-isolated vector writes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC
|
||||
from pathlib import Path
|
||||
|
||||
from tht.evidence.corpus.chunk import ChunkPolicy, chunk
|
||||
from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
|
||||
from tht.evidence.corpus.normalize import normalize
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
import tht.evidence.acquisition as evidence_acquisition
|
||||
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
from tht.ports.vector import VectorStore, VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
from tht.jobs.models import JobSpec
|
||||
from tht.jobs.runner import JobContext, StageArtifacts, run_job, seal_stage_artifacts
|
||||
|
||||
|
||||
EVIDENCE_STAGE_IDS = (
|
||||
"discover",
|
||||
"acquire_normalize_chunk",
|
||||
"embed",
|
||||
"vector_upsert",
|
||||
"stage_validate",
|
||||
"publish",
|
||||
"retention_cleanup",
|
||||
)
|
||||
|
||||
|
||||
class PipelineError(RuntimeError):
|
||||
"""Credential-free failure at the preprocessing boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PipelineResult:
|
||||
status: str
|
||||
generation: str | None
|
||||
published: bool
|
||||
changed: tuple[str, ...]
|
||||
unchanged: tuple[str, ...]
|
||||
removed: tuple[str, ...]
|
||||
manifest: CorpusManifest = field(repr=False)
|
||||
run_id: str | None = None
|
||||
resumed_from: str | None = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
counts = {
|
||||
"changed": len(self.changed),
|
||||
"unchanged": len(self.unchanged),
|
||||
"removed": len(self.removed),
|
||||
}
|
||||
return (
|
||||
f"PipelineResult(status={self.status!r}, generation={self.generation!r}, "
|
||||
f"published={self.published!r}, counts={counts!r}, "
|
||||
f"run_id={self.run_id!r}, resumed_from={self.resumed_from!r})"
|
||||
)
|
||||
|
||||
def model_dump(self, mode=None):
|
||||
def bounded(values: tuple[str, ...]) -> list[str]:
|
||||
return [value[:200] for value in values[:100]]
|
||||
|
||||
return {
|
||||
"status": self.status,
|
||||
"generation": self.generation,
|
||||
"published": self.published,
|
||||
"changed": bounded(self.changed),
|
||||
"unchanged": bounded(self.unchanged),
|
||||
"removed": bounded(self.removed),
|
||||
"counts": {
|
||||
"changed": len(self.changed),
|
||||
"unchanged": len(self.unchanged),
|
||||
"removed": len(self.removed),
|
||||
"documents": len(self.manifest.documents),
|
||||
"chunks": len(self.manifest.chunks),
|
||||
},
|
||||
"manifest_id": self.manifest.manifest_id,
|
||||
"run_id": self.run_id,
|
||||
"resumed_from": self.resumed_from,
|
||||
}
|
||||
|
||||
|
||||
def _fingerprint(value) -> str:
|
||||
payload = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||
return "sha256:" + hashlib.sha256(payload.encode()).hexdigest()
|
||||
|
||||
|
||||
def _canonical_json(value):
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _canonical_json(value[key]) for key in sorted(value)}
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return [_canonical_json(child) for child in value]
|
||||
return value
|
||||
|
||||
|
||||
def _source_snapshot(discovered) -> dict[str, dict]:
|
||||
snapshot = {}
|
||||
for _, item in discovered:
|
||||
modified_at = item.modified_at.astimezone(UTC) if item.modified_at else None
|
||||
metadata = _canonical_json(item.metadata)
|
||||
snapshot[item.source_id] = {
|
||||
"source_id": item.source_id,
|
||||
"uri": item.uri,
|
||||
"fingerprint": item.fingerprint,
|
||||
"modified_at": modified_at.isoformat().replace("+00:00", "Z") if modified_at else None,
|
||||
"metadata": metadata,
|
||||
"media_type": metadata.get("media_type"),
|
||||
"size": metadata.get("size"),
|
||||
}
|
||||
return snapshot
|
||||
|
||||
|
||||
class CorpusPipeline:
|
||||
def __init__(
|
||||
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
|
||||
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
|
||||
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
|
||||
workspace_id: str | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.sources = sources
|
||||
self.embedder = embedder
|
||||
self.vector_store = vector_store
|
||||
self.embedding_model = embedding_model
|
||||
self.embedding_dimensions = embedding_dimensions
|
||||
self.chunk_policy = chunk_policy
|
||||
self.pipeline_version = pipeline_version
|
||||
if isinstance(retain_published_generations, bool) or retain_published_generations < 1:
|
||||
raise ValueError("retain_published_generations must be at least 1")
|
||||
self.retain_published_generations = retain_published_generations
|
||||
self.workspace_id = workspace_id
|
||||
|
||||
def _assert_workspace_binding(self) -> None:
|
||||
manifest = self.store.active_manifest()
|
||||
if manifest is None:
|
||||
if self.workspace_id is None:
|
||||
self.workspace_id = "default"
|
||||
return
|
||||
persisted = manifest.metadata.get("workspace_id")
|
||||
if not isinstance(persisted, str) or re.fullmatch(
|
||||
r"[a-z][a-z0-9_-]{0,63}", persisted
|
||||
) is None:
|
||||
raise PipelineError(
|
||||
"corpus workspace ownership is missing or invalid; use a new corpus root or explicit rebuild"
|
||||
)
|
||||
if self.workspace_id is None and isinstance(persisted, str):
|
||||
self.workspace_id = persisted
|
||||
return
|
||||
if persisted != self.workspace_id:
|
||||
raise PipelineError(
|
||||
"corpus belongs to a different workspace; use a new corpus root or explicit rebuild"
|
||||
)
|
||||
|
||||
def _protected_generations(self, workspace_root: Path) -> set[str]:
|
||||
protected = {value for value in (self.store.active_generation(),) if value}
|
||||
runs = workspace_root / ".tht-jobs" / "evidence" / "runs"
|
||||
for checkpoint in runs.glob("*/checkpoint.json") if runs.exists() else ():
|
||||
try:
|
||||
state = json.loads(checkpoint.read_text(encoding="utf-8"))
|
||||
if state.get("status") not in {"running", "failed"}:
|
||||
continue
|
||||
plan = checkpoint.parent / "artifacts" / "plan.json"
|
||||
generation = json.loads(plan.read_text(encoding="utf-8")).get("generation")
|
||||
if isinstance(generation, str):
|
||||
protected.add(generation)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return protected
|
||||
|
||||
def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
|
||||
with self.store.writer_lock():
|
||||
return self._gc(workspace_root=workspace_root, dry_run=dry_run)
|
||||
|
||||
def _gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
|
||||
self._assert_workspace_binding()
|
||||
published = self.store.published_generations()
|
||||
list_vectors = getattr(self.vector_store, "list_evidence_generations", None)
|
||||
vector_generations = set(list_vectors("evidence", self.workspace_id)) if list_vectors else set()
|
||||
generations = sorted(set(published) | vector_generations)
|
||||
job_protected = self._protected_generations(workspace_root)
|
||||
active = self.store.active_generation()
|
||||
rollback_count = self.retain_published_generations - 1
|
||||
rollback = [generation for generation in published if generation != active]
|
||||
keep = ({active} if active else set()) | set(rollback[-rollback_count:] if rollback_count else ())
|
||||
fs_keep = keep | job_protected
|
||||
vector_protected = set(fs_keep)
|
||||
for generation in fs_keep:
|
||||
try:
|
||||
manifest = self.store.manifest(generation)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
vector_protected.update(
|
||||
value for value in manifest.metadata.get("document_generations", {}).values()
|
||||
if isinstance(value, str)
|
||||
)
|
||||
evicted, failures = [], []
|
||||
filesystem_generations = set(self.store.list_generations())
|
||||
for generation in generations:
|
||||
purge_vector = generation not in vector_protected
|
||||
purge_filesystem = generation in filesystem_generations and generation not in fs_keep
|
||||
if not purge_vector and not purge_filesystem:
|
||||
continue
|
||||
if dry_run:
|
||||
evicted.append(generation)
|
||||
continue
|
||||
if purge_vector:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
failures.append({"generation": generation, "error": "vector cleanup failed"})
|
||||
continue
|
||||
try:
|
||||
if purge_filesystem:
|
||||
self.store.discard(generation)
|
||||
evicted.append(generation)
|
||||
except Exception:
|
||||
failures.append({"generation": generation, "error": "filesystem cleanup failed"})
|
||||
return {"status": "partial" if failures else "succeeded", "dry_run": dry_run,
|
||||
"active_generation": self.store.active_generation(), "evicted": evicted,
|
||||
"protected": sorted(vector_protected), "failures": failures}
|
||||
|
||||
def _discover(self) -> list[tuple[EvidenceSource, SourceObject]]:
|
||||
discovered = []
|
||||
seen = set()
|
||||
for source in self.sources:
|
||||
for item in evidence_acquisition.discover(source):
|
||||
if item.source_id in seen:
|
||||
raise PipelineError("duplicate Evidence source identity")
|
||||
seen.add(item.source_id)
|
||||
discovered.append((source, item))
|
||||
return sorted(discovered, key=lambda pair: pair[1].source_id)
|
||||
|
||||
def run(self, *, dry_run: bool = False, resume: str | None = None) -> PipelineResult:
|
||||
with self.store.writer_lock():
|
||||
self._assert_workspace_binding()
|
||||
return self._run(dry_run=dry_run, resume=resume)
|
||||
|
||||
def run_as_job(self, **kwargs) -> PipelineResult:
|
||||
self.workspace_id = kwargs["workspace_id"]
|
||||
with self.store.writer_lock():
|
||||
self._assert_workspace_binding()
|
||||
return self._run_as_job(**kwargs)
|
||||
|
||||
def _run_as_job(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
workspace_root: Path,
|
||||
config_fingerprint: str,
|
||||
input_fingerprint: str,
|
||||
dry_run: bool = False,
|
||||
resume_run_id: str | None = None,
|
||||
after_stage_return=None,
|
||||
) -> PipelineResult:
|
||||
"""Execute preprocessing through the durable shared job envelope."""
|
||||
discovered = self._discover()
|
||||
discovered_fingerprint = _fingerprint(
|
||||
{item.source_id: item.fingerprint for _, item in discovered}
|
||||
)
|
||||
source_snapshot = _source_snapshot(discovered)
|
||||
source_by_id = {item.source_id: (source, item) for source, item in discovered}
|
||||
compatibility = _fingerprint({
|
||||
"pipeline": self.pipeline_version,
|
||||
"model": self.embedding_model,
|
||||
"dimensions": self.embedding_dimensions,
|
||||
"chunk_policy": asdict(self.chunk_policy),
|
||||
})
|
||||
job_binding = {
|
||||
"config_fingerprint": config_fingerprint,
|
||||
"input_fingerprint": input_fingerprint,
|
||||
"compatibility_fingerprint": compatibility,
|
||||
"pipeline_version": self.pipeline_version,
|
||||
"chunk_policy_version": self.chunk_policy.version,
|
||||
"embedding_model": self.embedding_model,
|
||||
"embedding_dimensions": self.embedding_dimensions,
|
||||
}
|
||||
previous = self.store.active_manifest()
|
||||
|
||||
def document_sources(manifest: CorpusManifest) -> dict[str, dict]:
|
||||
return {
|
||||
document.document_id: {
|
||||
"document_id": document.document_id,
|
||||
"source_id": document.source_id,
|
||||
"source_uri": document.source_uri,
|
||||
"source_fingerprint": document.source_fingerprint,
|
||||
"modified_at": (
|
||||
document.modified_at.isoformat().replace("+00:00", "Z")
|
||||
if document.modified_at else None
|
||||
),
|
||||
"source_metadata": _canonical_json(document.metadata.get("source")),
|
||||
"media_type": document.media_type,
|
||||
"content_hash": document.content_hash,
|
||||
"pipeline_version": document.pipeline_version,
|
||||
}
|
||||
for document in manifest.documents
|
||||
}
|
||||
|
||||
def active_assets_are_valid(manifest: CorpusManifest | None) -> bool:
|
||||
if manifest is None or manifest.metadata.get("workspace_id") != workspace_id:
|
||||
return False
|
||||
actual_documents = {document.source_id: document for document in manifest.documents}
|
||||
persisted_snapshot = _canonical_json(manifest.metadata.get("source_snapshot"))
|
||||
if (
|
||||
not isinstance(persisted_snapshot, dict)
|
||||
or set(persisted_snapshot) != set(actual_documents)
|
||||
or manifest.metadata.get("compatibility_fingerprint") != compatibility
|
||||
or _canonical_json(manifest.metadata.get("document_sources"))
|
||||
!= document_sources(manifest)
|
||||
):
|
||||
return False
|
||||
for source_id, document in actual_documents.items():
|
||||
source_payload = persisted_snapshot[source_id]
|
||||
source = SourceObject.model_validate({
|
||||
"source_id": source_payload["source_id"],
|
||||
"uri": source_payload["uri"],
|
||||
"fingerprint": source_payload["fingerprint"],
|
||||
"modified_at": source_payload["modified_at"],
|
||||
"metadata": source_payload["metadata"],
|
||||
})
|
||||
content = self.store.read_document(document.document_id, manifest.manifest_id)
|
||||
expected_uri = canonical_provenance_uri(source.uri)
|
||||
expected_id = "doc:" + hashlib.sha256(
|
||||
f"{source.source_id}\n{expected_uri}".encode()
|
||||
).hexdigest()
|
||||
expected_media_type = source_payload.get("media_type")
|
||||
if (
|
||||
content != document.content
|
||||
or document.document_id != expected_id
|
||||
or document.source_id != source.source_id
|
||||
or document.source_uri != expected_uri
|
||||
or document.source_fingerprint != source.fingerprint
|
||||
or document.modified_at != source.modified_at
|
||||
or _canonical_json(document.metadata.get("source"))
|
||||
!= _canonical_json(source.metadata)
|
||||
or (
|
||||
isinstance(expected_media_type, str)
|
||||
and document.media_type != expected_media_type
|
||||
)
|
||||
or document.pipeline_version != self.pipeline_version
|
||||
):
|
||||
return False
|
||||
expected_chunks = tuple(
|
||||
part for document in manifest.documents for part in chunk(document, self.chunk_policy)
|
||||
)
|
||||
if any(document.content and not chunk(document, self.chunk_policy)
|
||||
for document in manifest.documents):
|
||||
return False
|
||||
if _canonical_json([part.model_dump(mode="json") for part in manifest.chunks]) != (
|
||||
_canonical_json([part.model_dump(mode="json") for part in expected_chunks])
|
||||
):
|
||||
return False
|
||||
generations = manifest.metadata.get("document_generations")
|
||||
if not isinstance(generations, Mapping):
|
||||
return False
|
||||
health = self.vector_store.health()
|
||||
if (
|
||||
not health.ok
|
||||
or health.dimension_compatible is not True
|
||||
or health.expected_dimension != self.embedding_dimensions
|
||||
or health.observed_dimensions != (self.embedding_dimensions,)
|
||||
):
|
||||
return False
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
for part in expected_chunks:
|
||||
generation = generations.get(part.document_id)
|
||||
if not isinstance(generation, str):
|
||||
return False
|
||||
record_id = f"{workspace_id}:{generation}:{part.chunk_id}"
|
||||
if existing.get(record_id) != part.content_hash:
|
||||
return False
|
||||
return True
|
||||
|
||||
try:
|
||||
active_assets_valid = active_assets_are_valid(previous)
|
||||
except Exception:
|
||||
active_assets_valid = False
|
||||
reusable = (
|
||||
active_assets_valid
|
||||
and _canonical_json(previous.metadata.get("source_snapshot")) == source_snapshot
|
||||
and _canonical_json(previous.metadata.get("job_binding")) == job_binding
|
||||
)
|
||||
if not dry_run and resume_run_id is None and reusable:
|
||||
return PipelineResult(
|
||||
"succeeded", previous.manifest_id, False, (),
|
||||
tuple(sorted(item.source_id for _, item in discovered)), (), previous,
|
||||
)
|
||||
spec = JobSpec(
|
||||
workspace_id=workspace_id,
|
||||
job_type="evidence",
|
||||
workspace_root=workspace_root,
|
||||
spec_version="jobs-v1",
|
||||
pipeline_version=self.pipeline_version,
|
||||
config_fingerprint=config_fingerprint,
|
||||
input_fingerprint=_fingerprint([input_fingerprint, discovered_fingerprint]),
|
||||
stage_ids=EVIDENCE_STAGE_IDS,
|
||||
dry_run=dry_run,
|
||||
resume_run_id=resume_run_id,
|
||||
)
|
||||
|
||||
def artifact(context: JobContext, name: str) -> Path:
|
||||
root = context.run_dir / "artifacts"
|
||||
root.mkdir(exist_ok=True)
|
||||
return root / name
|
||||
|
||||
def write(context: JobContext, name: str, value) -> None:
|
||||
artifact(context, name).write_text(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")), encoding="utf-8"
|
||||
)
|
||||
|
||||
def read(context: JobContext, name: str):
|
||||
try:
|
||||
return json.loads(artifact(context, name).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as error:
|
||||
raise PipelineError("preprocessing checkpoint artifact is corrupt") from error
|
||||
|
||||
def discover_stage(context: JobContext) -> None:
|
||||
previous = self.store.active_manifest()
|
||||
prior = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
||||
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
|
||||
previous_snapshot = (
|
||||
_canonical_json(previous.metadata.get("source_snapshot")) if previous else {}
|
||||
)
|
||||
rebuild = bool(previous and not active_assets_valid)
|
||||
changed = sorted(
|
||||
item.source_id for _, item in discovered
|
||||
if rebuild or item.source_id not in prior
|
||||
or previous_snapshot.get(item.source_id) != source_snapshot[item.source_id]
|
||||
)
|
||||
unchanged = sorted(set(fingerprints) - set(changed))
|
||||
removed = sorted(set(prior) - set(fingerprints))
|
||||
write(context, "plan.json", {
|
||||
"generation": f"gen:{context.run_id}",
|
||||
"compatibility": compatibility,
|
||||
"job_binding": job_binding,
|
||||
"source_snapshot": source_snapshot,
|
||||
"fingerprints": fingerprints,
|
||||
"changed": changed,
|
||||
"unchanged": unchanged,
|
||||
"removed": removed,
|
||||
"previous": previous.model_dump(mode="json") if previous else None,
|
||||
})
|
||||
return StageArtifacts(("plan.json",))
|
||||
|
||||
def acquire_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
plan = read(context, "plan.json")
|
||||
previous = CorpusManifest.model_validate(plan["previous"]) if plan["previous"] else None
|
||||
prior = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
||||
documents = [prior[source_id] for source_id in plan["unchanged"]]
|
||||
for source_id in plan["changed"]:
|
||||
source, item = source_by_id[source_id]
|
||||
documents.append(normalize(
|
||||
evidence_acquisition.acquire(source, item), self.pipeline_version,
|
||||
))
|
||||
documents.sort(key=lambda value: value.source_id)
|
||||
chunks = [part for document in documents for part in chunk(document, self.chunk_policy)]
|
||||
previous_generations = dict(previous.metadata.get("document_generations", {})) if previous else {}
|
||||
changed = set(plan["changed"])
|
||||
generations = {
|
||||
document.document_id: (
|
||||
plan["generation"] if document.source_id in changed
|
||||
else previous_generations.get(document.document_id, previous.vector_generation)
|
||||
) for document in documents
|
||||
}
|
||||
manifest = CorpusManifest(
|
||||
pipeline_version=self.pipeline_version,
|
||||
embedding_model=self.embedding_model,
|
||||
embedding_dimensions=self.embedding_dimensions,
|
||||
vector_generation=plan["generation"],
|
||||
documents=tuple(documents), chunks=tuple(chunks),
|
||||
metadata={
|
||||
"workspace_id": self.workspace_id,
|
||||
"compatibility_fingerprint": compatibility,
|
||||
"job_binding": plan["job_binding"],
|
||||
"source_snapshot": plan["source_snapshot"],
|
||||
"document_sources": {
|
||||
document.document_id: {
|
||||
"document_id": document.document_id,
|
||||
"source_id": document.source_id,
|
||||
"source_uri": document.source_uri,
|
||||
"source_fingerprint": document.source_fingerprint,
|
||||
"modified_at": (
|
||||
document.modified_at.isoformat().replace("+00:00", "Z")
|
||||
if document.modified_at else None
|
||||
),
|
||||
"source_metadata": _canonical_json(
|
||||
document.metadata.get("source")
|
||||
),
|
||||
"media_type": document.media_type,
|
||||
"content_hash": document.content_hash,
|
||||
"pipeline_version": document.pipeline_version,
|
||||
}
|
||||
for document in documents
|
||||
},
|
||||
"fingerprints": plan["fingerprints"],
|
||||
"removed": plan["removed"],
|
||||
"document_generations": generations,
|
||||
},
|
||||
)
|
||||
write(context, "manifest.json", manifest.model_dump(mode="json"))
|
||||
return StageArtifacts(("manifest.json",))
|
||||
|
||||
def embed_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
|
||||
parts = [part for part in manifest.chunks if part.document_id in changed_docs]
|
||||
embeddings = self.embedder.embed_documents([part.content for part in parts])
|
||||
if len(embeddings) != len(parts) or any(
|
||||
len(vector) != self.embedding_dimensions for vector in embeddings
|
||||
):
|
||||
raise PipelineError("embedding output is incompatible")
|
||||
write(context, "embeddings.json", embeddings)
|
||||
return StageArtifacts(("embeddings.json",))
|
||||
|
||||
def records(context: JobContext):
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
|
||||
parts = [part for part in manifest.chunks if part.document_id in changed_docs]
|
||||
embeddings = read(context, "embeddings.json")
|
||||
return [self._vector_record(part, vector, plan["generation"], self.workspace_id)
|
||||
for part, vector in zip(parts, embeddings, strict=True)]
|
||||
|
||||
def compensate(context: JobContext) -> None:
|
||||
generation = read(context, "plan.json")["generation"]
|
||||
if self.store.active_generation() != generation:
|
||||
self.store.discard(generation)
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
write(context, "compensated.json", {"generation": generation})
|
||||
|
||||
def rotate_compensated_generation(context: JobContext) -> None:
|
||||
marker = artifact(context, "compensated.json")
|
||||
if not marker.exists():
|
||||
return
|
||||
plan = read(context, "plan.json")
|
||||
old = plan["generation"]
|
||||
plan["generation"] = f"gen:{uuid.uuid4().hex}"
|
||||
write(context, "plan.json", plan)
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
changed = set(plan["changed"])
|
||||
generations = dict(manifest.metadata["document_generations"])
|
||||
for document in manifest.documents:
|
||||
if document.source_id in changed and generations.get(document.document_id) == old:
|
||||
generations[document.document_id] = plan["generation"]
|
||||
manifest_payload = manifest.model_dump(mode="json")
|
||||
manifest_payload["metadata"]["document_generations"] = generations
|
||||
manifest_payload["vector_generation"] = plan["generation"]
|
||||
manifest = CorpusManifest.model_validate(manifest_payload)
|
||||
write(context, "manifest.json", manifest.model_dump(mode="json"))
|
||||
marker.unlink()
|
||||
|
||||
def vector_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
rotate_compensated_generation(context)
|
||||
values = records(context)
|
||||
write(context, "vector-intent.json", {
|
||||
"generation": read(context, "plan.json")["generation"],
|
||||
"records": {value.record.id: value.content_hash for value in values},
|
||||
})
|
||||
seal_stage_artifacts(
|
||||
context, "vector_upsert",
|
||||
("plan.json", "manifest.json", "vector-intent.json"), spec,
|
||||
)
|
||||
try:
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
missing = [
|
||||
value for value in values
|
||||
if existing.get(value.record.id) != value.content_hash
|
||||
]
|
||||
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
|
||||
|
||||
def stage_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
recovered = False
|
||||
try:
|
||||
if artifact(context, "compensated.json").exists():
|
||||
recovered = True
|
||||
rotate_compensated_generation(context)
|
||||
values = records(context)
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
|
||||
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
if not self.store.generation_path(plan["generation"]).exists():
|
||||
self.store.stage(
|
||||
manifest, {doc.document_id: doc.content for doc in manifest.documents},
|
||||
generation=plan["generation"],
|
||||
)
|
||||
self.store.manifest(plan["generation"])
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
return StageArtifacts(
|
||||
("plan.json", "manifest.json", "vector-intent.json") if recovered else ()
|
||||
)
|
||||
|
||||
def publish_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
if artifact(context, "compensated.json").exists():
|
||||
rotate_compensated_generation(context)
|
||||
values = records(context)
|
||||
try:
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
|
||||
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
generation = read(context, "plan.json")["generation"]
|
||||
try:
|
||||
if not self.store.generation_path(generation).exists():
|
||||
self.store.stage(
|
||||
manifest, {doc.document_id: doc.content for doc in manifest.documents},
|
||||
generation=generation,
|
||||
)
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
generation = read(context, "plan.json")["generation"]
|
||||
try:
|
||||
self.store.publish(generation)
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
|
||||
|
||||
def retention_stage(context: JobContext) -> None:
|
||||
if not context.dry_run:
|
||||
self.gc(workspace_root=workspace_root)
|
||||
|
||||
report = run_job(spec, [
|
||||
discover_stage, acquire_stage, embed_stage, vector_stage,
|
||||
stage_stage, publish_stage, retention_stage,
|
||||
], after_stage_return=after_stage_return)
|
||||
run_dir = workspace_root / ".tht-jobs" / "evidence" / "runs" / report.run_id
|
||||
plan = json.loads((run_dir / "artifacts" / "plan.json").read_text())
|
||||
if dry_run:
|
||||
manifest = self.store.active_manifest() or CorpusManifest(pipeline_version=self.pipeline_version)
|
||||
generation = None
|
||||
published = False
|
||||
elif report.status == "succeeded":
|
||||
generation = plan["generation"]
|
||||
manifest = self.store.manifest(generation)
|
||||
published = True
|
||||
else:
|
||||
generation = plan["generation"]
|
||||
manifest_path = run_dir / "artifacts" / "manifest.json"
|
||||
manifest = (CorpusManifest.model_validate_json(manifest_path.read_text())
|
||||
if manifest_path.exists() else CorpusManifest(pipeline_version=self.pipeline_version))
|
||||
published = False
|
||||
return PipelineResult(
|
||||
report.status, generation, published, tuple(plan["changed"]),
|
||||
tuple(plan["unchanged"]), tuple(plan["removed"]), manifest,
|
||||
report.run_id, report.resumed_from,
|
||||
)
|
||||
|
||||
def _run(self, *, dry_run: bool = False, resume: str | None = None) -> PipelineResult:
|
||||
generation = None
|
||||
vector_written = False
|
||||
previous = self.store.active_manifest()
|
||||
try:
|
||||
discovered = self._discover()
|
||||
except Exception as error:
|
||||
raise PipelineError("Evidence discovery failed") from error
|
||||
prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
||||
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
|
||||
compatibility = _fingerprint({
|
||||
"pipeline": self.pipeline_version, "model": self.embedding_model,
|
||||
"dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy),
|
||||
})
|
||||
previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None
|
||||
rebuild = previous is not None and compatibility != previous_compatibility
|
||||
changed = tuple(item.source_id for _, item in discovered if rebuild or prior_documents.get(item.source_id) is None or prior_documents[item.source_id].source_fingerprint != item.fingerprint)
|
||||
unchanged = tuple(item.source_id for _, item in discovered if item.source_id not in changed)
|
||||
removed = tuple(sorted(set(prior_documents) - set(fingerprints)))
|
||||
if dry_run:
|
||||
manifest = previous or CorpusManifest(pipeline_version=self.pipeline_version)
|
||||
return PipelineResult("succeeded", None, False, changed, unchanged, removed, manifest)
|
||||
if previous is not None and not changed and not removed:
|
||||
return PipelineResult(
|
||||
"succeeded", previous.manifest_id, False, changed, unchanged, removed, previous
|
||||
)
|
||||
|
||||
documents: list[CanonicalDocument] = [prior_documents[source_id] for source_id in unchanged]
|
||||
changed_set = set(changed)
|
||||
try:
|
||||
for source, item in discovered:
|
||||
if item.source_id in changed_set:
|
||||
documents.append(normalize(
|
||||
evidence_acquisition.acquire(source, item), self.pipeline_version,
|
||||
))
|
||||
documents.sort(key=lambda document: document.source_id)
|
||||
chunks: list[CanonicalChunk] = []
|
||||
for document in documents:
|
||||
chunks.extend(chunk(document, self.chunk_policy))
|
||||
generation = resume or f"gen:{uuid.uuid4().hex}"
|
||||
previous_generations = dict(previous.metadata.get("document_generations", {})) if previous else {}
|
||||
document_generations = {
|
||||
document.document_id: (
|
||||
generation if document.source_id in changed_set
|
||||
else previous_generations.get(document.document_id, previous.vector_generation)
|
||||
)
|
||||
for document in documents
|
||||
}
|
||||
manifest = CorpusManifest(
|
||||
pipeline_version=self.pipeline_version,
|
||||
embedding_model=self.embedding_model,
|
||||
embedding_dimensions=self.embedding_dimensions,
|
||||
vector_generation=generation,
|
||||
documents=tuple(documents), chunks=tuple(chunks),
|
||||
metadata={
|
||||
"workspace_id": self.workspace_id,
|
||||
"compatibility_fingerprint": compatibility,
|
||||
"fingerprints": fingerprints,
|
||||
"removed": list(removed),
|
||||
"document_generations": document_generations,
|
||||
},
|
||||
)
|
||||
changed_documents = {document.document_id for document in documents if document.source_id in changed_set}
|
||||
changed_chunks = [part for part in chunks if part.document_id in changed_documents]
|
||||
embeddings = self.embedder.embed_documents([part.content for part in changed_chunks])
|
||||
if len(embeddings) != len(changed_chunks):
|
||||
raise PipelineError("embedding count mismatch")
|
||||
if any(len(vector) != self.embedding_dimensions for vector in embeddings):
|
||||
raise PipelineError("embedding dimension mismatch")
|
||||
records = [self._vector_record(part, vector, generation, self.workspace_id) for part, vector in zip(changed_chunks, embeddings, strict=True)]
|
||||
if records:
|
||||
written = self.vector_store.upsert("evidence", records)
|
||||
vector_written = True
|
||||
if written != len(records):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
generation_path = self.store.generation_path(generation)
|
||||
if resume is not None and generation_path.exists():
|
||||
staged_manifest = self.store.manifest(generation)
|
||||
expected = manifest.model_dump(mode="json", exclude={"created_at", "manifest_id"})
|
||||
actual = staged_manifest.model_dump(mode="json", exclude={"created_at", "manifest_id"})
|
||||
actual["metadata"].pop("files", None)
|
||||
if actual != expected:
|
||||
raise PipelineError("resume generation is incompatible")
|
||||
staged = generation
|
||||
else:
|
||||
staged = self.store.stage(
|
||||
manifest, {document.document_id: document.content for document in documents},
|
||||
generation=generation,
|
||||
)
|
||||
self.store.publish(staged)
|
||||
self.gc(workspace_root=self.store.root.parent)
|
||||
except PipelineError:
|
||||
self._compensate(generation, vector_written)
|
||||
raise
|
||||
except Exception as error:
|
||||
self._compensate(generation, vector_written)
|
||||
raise PipelineError("Evidence preprocessing failed") from error
|
||||
return PipelineResult("succeeded", generation, True, changed, unchanged, removed, self.store.manifest(generation))
|
||||
|
||||
def _compensate(self, generation: str | None, vector_written: bool) -> None:
|
||||
if generation is None:
|
||||
return
|
||||
try:
|
||||
self.store.discard(generation)
|
||||
except Exception:
|
||||
pass
|
||||
if vector_written:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _vector_record(
|
||||
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
|
||||
):
|
||||
record = VectorRecord(
|
||||
id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
|
||||
kind="evidence", ref=chunk.document_id,
|
||||
title=str(chunk.metadata.get("title", "")), content=chunk.content,
|
||||
metadata={
|
||||
**dict(chunk.metadata), "document_id": chunk.document_id,
|
||||
"workspace_id": workspace_id,
|
||||
"source_uri": chunk.source_uri, "ordinal": chunk.ordinal,
|
||||
"vector_generation": generation,
|
||||
},
|
||||
)
|
||||
return VectorWriteRecord(record=record, embedding=embedding, content_hash=chunk.content_hash)
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Evidence-owned immutable corpus generations and an atomic ACTIVE pointer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import fcntl
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import shutil
|
||||
import uuid
|
||||
import hashlib
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from contextlib import contextmanager
|
||||
|
||||
from tht.evidence.corpus.models import CorpusManifest
|
||||
|
||||
|
||||
_GENERATION = re.compile(r"^gen:[0-9a-f]{32}$")
|
||||
|
||||
|
||||
class UnsafeCorpusPath(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _atomic_write(path: Path, payload: bytes) -> None:
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
class CorpusStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
self.active_path = self.root / "ACTIVE"
|
||||
self._replace = os.replace
|
||||
self._fsync_directory = self._sync_root
|
||||
self._lock_state = threading.local()
|
||||
self._ensure_root()
|
||||
|
||||
def _ensure_root(self) -> None:
|
||||
if self.root.is_symlink():
|
||||
raise UnsafeCorpusPath("corpus root must not be a symlink")
|
||||
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
info = self.root.lstat()
|
||||
if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid():
|
||||
raise UnsafeCorpusPath("corpus root is unsafe")
|
||||
|
||||
@contextmanager
|
||||
def writer_lock(self):
|
||||
depth = getattr(self._lock_state, "depth", 0)
|
||||
if depth:
|
||||
self._lock_state.depth = depth + 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._lock_state.depth -= 1
|
||||
return
|
||||
lock_path = self.root / ".writer.lock"
|
||||
fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
|
||||
raise UnsafeCorpusPath("corpus writer lock is unsafe")
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
self._lock_state.depth = 1
|
||||
yield
|
||||
finally:
|
||||
self._lock_state.depth = 0
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
os.close(fd)
|
||||
|
||||
def generation_path(self, generation: str) -> Path:
|
||||
if not _GENERATION.fullmatch(generation):
|
||||
raise UnsafeCorpusPath("invalid corpus generation")
|
||||
path = self.root / generation.replace(":", "-")
|
||||
if path.is_symlink():
|
||||
raise UnsafeCorpusPath("generation must not be a symlink")
|
||||
return path
|
||||
|
||||
def stage(
|
||||
self,
|
||||
manifest: CorpusManifest,
|
||||
materialized: dict[str, str],
|
||||
*,
|
||||
generation: str | None = None,
|
||||
) -> str:
|
||||
generation = generation or f"gen:{uuid.uuid4().hex}"
|
||||
path = self.generation_path(generation)
|
||||
try:
|
||||
path.mkdir(mode=0o700)
|
||||
except FileExistsError:
|
||||
raise UnsafeCorpusPath("generation already exists") from None
|
||||
documents = path / "documents"
|
||||
documents.mkdir(mode=0o700)
|
||||
files: dict[str, str] = {}
|
||||
for document in manifest.documents:
|
||||
relative = f"documents/{document.document_id.removeprefix('doc:')}.md"
|
||||
_atomic_write(path / relative, materialized[document.document_id].encode("utf-8"))
|
||||
files[document.document_id] = relative
|
||||
payload = json.loads(manifest.model_dump_json())
|
||||
metadata = payload["metadata"]
|
||||
metadata["files"] = files
|
||||
payload.update({"manifest_id": generation, "metadata": metadata})
|
||||
staged = CorpusManifest.model_validate(payload)
|
||||
_atomic_write(path / "manifest.json", (staged.model_dump_json(indent=2) + "\n").encode())
|
||||
return generation
|
||||
|
||||
def publish(self, generation: str) -> str:
|
||||
manifest = self.manifest(generation)
|
||||
if manifest.manifest_id != generation:
|
||||
raise UnsafeCorpusPath("manifest generation mismatch")
|
||||
if self.active_generation() == generation:
|
||||
return generation
|
||||
previous = self.active_generation()
|
||||
published_marker = self.generation_path(generation) / "PUBLISHED"
|
||||
temporary = self.active_path.with_name(f".ACTIVE.{uuid.uuid4().hex}.tmp")
|
||||
replaced = False
|
||||
try:
|
||||
_atomic_write(temporary, (generation + "\n").encode())
|
||||
self._replace(temporary, self.active_path)
|
||||
replaced = True
|
||||
self._fsync_directory()
|
||||
_atomic_write(
|
||||
published_marker,
|
||||
(datetime.now(UTC).isoformat().replace("+00:00", "Z") + "\n").encode("ascii"),
|
||||
)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
if replaced:
|
||||
if previous is None:
|
||||
self.active_path.unlink(missing_ok=True)
|
||||
else:
|
||||
rollback = self.active_path.with_name(f".ACTIVE.rollback.{uuid.uuid4().hex}.tmp")
|
||||
_atomic_write(rollback, (previous + "\n").encode())
|
||||
self._replace(rollback, self.active_path)
|
||||
self._sync_root()
|
||||
raise
|
||||
return generation
|
||||
|
||||
def _sync_root(self) -> None:
|
||||
directory = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
|
||||
def active_generation(self) -> str | None:
|
||||
try:
|
||||
if self.active_path.is_symlink():
|
||||
raise UnsafeCorpusPath("ACTIVE must not be a symlink")
|
||||
value = self.active_path.read_text(encoding="ascii").strip()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if not _GENERATION.fullmatch(value):
|
||||
raise UnsafeCorpusPath("ACTIVE contains an invalid generation")
|
||||
return value
|
||||
|
||||
def manifest(self, generation: str) -> CorpusManifest:
|
||||
path = self.generation_path(generation)
|
||||
manifest_path = path / "manifest.json"
|
||||
if manifest_path.is_symlink():
|
||||
raise UnsafeCorpusPath("manifest must not be a symlink")
|
||||
return CorpusManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
|
||||
|
||||
def discard(self, generation: str) -> None:
|
||||
path = self.generation_path(generation)
|
||||
if path.exists():
|
||||
if path.is_symlink() or not stat.S_ISDIR(path.lstat().st_mode):
|
||||
raise UnsafeCorpusPath("generation cleanup target is unsafe")
|
||||
shutil.rmtree(path)
|
||||
|
||||
def active_manifest(self) -> CorpusManifest | None:
|
||||
generation = self.active_generation()
|
||||
return self.manifest(generation) if generation else None
|
||||
|
||||
def list_generations(self) -> list[str]:
|
||||
values = []
|
||||
for entry in self.root.iterdir():
|
||||
match = re.fullmatch(r"gen-([0-9a-f]{32})", entry.name)
|
||||
if match and not entry.is_symlink() and stat.S_ISDIR(entry.lstat().st_mode):
|
||||
values.append(f"gen:{match.group(1)}")
|
||||
return sorted(values, key=lambda value: self.generation_path(value).stat().st_mtime_ns)
|
||||
|
||||
def published_generations(self) -> list[str]:
|
||||
active = self.active_generation()
|
||||
published = []
|
||||
for generation in self.list_generations():
|
||||
path = self.generation_path(generation)
|
||||
marker = path / "PUBLISHED"
|
||||
if generation != active and not marker.is_file():
|
||||
continue
|
||||
try:
|
||||
manifest = self.manifest(generation)
|
||||
if manifest.manifest_id != generation:
|
||||
continue
|
||||
timestamp = marker.read_text(encoding="ascii").strip() if marker.is_file() else ""
|
||||
key = (timestamp or manifest.created_at.isoformat(), generation)
|
||||
published.append((key, generation))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return [generation for _, generation in sorted(published)]
|
||||
|
||||
def resolve_document(self, document_id: str, generation: str | None = None) -> Path | None:
|
||||
generation = generation or self.active_generation()
|
||||
if generation is None:
|
||||
return None
|
||||
manifest = self.manifest(generation)
|
||||
relative = manifest.metadata.get("files", {}).get(document_id)
|
||||
if not isinstance(relative, str):
|
||||
return None
|
||||
parts = Path(relative).parts
|
||||
if Path(relative).is_absolute() or parts[:1] != ("documents",) or len(parts) != 2:
|
||||
raise UnsafeCorpusPath("materialized document path is unsafe")
|
||||
return self.generation_path(generation) / relative
|
||||
|
||||
def read_document(self, document_id: str, generation: str | None = None) -> str | None:
|
||||
generation = generation or self.active_generation()
|
||||
if generation is None:
|
||||
return None
|
||||
manifest = self.manifest(generation)
|
||||
path = self.resolve_document(document_id, generation)
|
||||
document = next((item for item in manifest.documents if item.document_id == document_id), None)
|
||||
if path is None or document is None:
|
||||
return None
|
||||
generation_fd = os.open(self.generation_path(generation), os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
documents_fd = fd = None
|
||||
try:
|
||||
documents_fd = os.open("documents", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=generation_fd)
|
||||
fd = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=documents_fd)
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
|
||||
raise UnsafeCorpusPath("materialized document is unsafe")
|
||||
payload = os.read(fd, info.st_size + 1)
|
||||
if len(payload) != info.st_size or "sha256:" + hashlib.sha256(payload).hexdigest() != document.content_hash:
|
||||
raise UnsafeCorpusPath("materialized document hash mismatch")
|
||||
return payload.decode("utf-8")
|
||||
except (OSError, UnicodeError) as error:
|
||||
raise UnsafeCorpusPath("materialized document read failed") from error
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
if documents_fd is not None:
|
||||
os.close(documents_fd)
|
||||
os.close(generation_fd)
|
||||
|
||||
def materialize_document(
|
||||
self, document_id: str, destination: Path, generation: str | None = None,
|
||||
) -> Path | None:
|
||||
content = self.read_document(document_id, generation)
|
||||
if content is None:
|
||||
return None
|
||||
destination = Path(destination)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
_atomic_write(destination, content.encode("utf-8"))
|
||||
destination.chmod(0o400)
|
||||
return destination
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from tht.corpus.chunk import ChunkPolicy
|
||||
from tht.corpus.pipeline import CorpusPipeline
|
||||
from tht.corpus.store import CorpusStore
|
||||
from tht.evidence.corpus.chunk import ChunkPolicy
|
||||
from tht.evidence.corpus.pipeline import CorpusPipeline
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
from tht.evidence.contracts import EvidenceSource
|
||||
from tht.ports.vector import VectorStore
|
||||
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Evidence-owned runtime lookup bound to the atomically active corpus generation."""
|
||||
|
||||
import re
|
||||
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
|
||||
|
||||
class CorpusWorkspaceMismatchError(RuntimeError):
|
||||
"""The configured workspace does not own the persisted corpus."""
|
||||
|
||||
|
||||
class ActiveEvidenceSearcher:
|
||||
"""Searcher facade that enforces ACTIVE generation predicates before LIMIT."""
|
||||
|
||||
def __init__(self, corpus: CorpusStore, delegate, expected_workspace_id: str | None = None):
|
||||
self.corpus = corpus
|
||||
self.delegate = delegate
|
||||
self.expected_workspace_id = expected_workspace_id
|
||||
|
||||
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
|
||||
requested = set(kinds) if kinds is not None else {
|
||||
"schema_table", "schema_column", "evidence", "memory", "solved_question",
|
||||
}
|
||||
include_evidence = "evidence" in requested
|
||||
other_kinds = sorted(requested - {"evidence"})
|
||||
with self.corpus.writer_lock():
|
||||
manifest = self.corpus.active_manifest()
|
||||
persisted_workspace = manifest.metadata.get("workspace_id") if manifest else None
|
||||
if manifest is not None and (
|
||||
not isinstance(persisted_workspace, str)
|
||||
or re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", persisted_workspace) is None
|
||||
):
|
||||
raise CorpusWorkspaceMismatchError(
|
||||
"corpus workspace ownership is missing or invalid; use a new corpus root or rebuild"
|
||||
)
|
||||
if manifest is not None and self.expected_workspace_id is not None and (
|
||||
persisted_workspace != self.expected_workspace_id
|
||||
):
|
||||
raise CorpusWorkspaceMismatchError(
|
||||
"corpus belongs to a different workspace; use a new corpus root or rebuild"
|
||||
)
|
||||
if not include_evidence:
|
||||
kwargs = {"top_n": top_n, "kinds": kinds}
|
||||
if metadata_filter is not None:
|
||||
kwargs["metadata_filter"] = metadata_filter
|
||||
return self.delegate.search(embedding, **kwargs)
|
||||
hits = []
|
||||
if other_kinds:
|
||||
kwargs = {"top_n": top_n, "kinds": other_kinds}
|
||||
if metadata_filter is not None:
|
||||
kwargs["metadata_filter"] = metadata_filter
|
||||
hits.extend(self.delegate.search(embedding, **kwargs))
|
||||
if include_evidence:
|
||||
workspace_id = manifest.metadata.get("workspace_id") if manifest else None
|
||||
if manifest is not None and isinstance(workspace_id, str):
|
||||
by_generation: dict[str, list[str]] = {}
|
||||
mapping = dict(manifest.metadata.get("document_generations", {}))
|
||||
for document in manifest.documents:
|
||||
generation = mapping.get(document.document_id, manifest.vector_generation)
|
||||
if generation:
|
||||
by_generation.setdefault(generation, []).append(document.document_id)
|
||||
for generation, document_ids in sorted(by_generation.items()):
|
||||
hits.extend(self.delegate.search(
|
||||
embedding, top_n=top_n, kinds=["evidence"],
|
||||
metadata_filter={
|
||||
"vector_generation": generation,
|
||||
"document_ids": sorted(document_ids),
|
||||
"workspace_id": workspace_id,
|
||||
},
|
||||
))
|
||||
return sorted(hits, key=lambda hit: (-hit.similarity, hit.id))[:top_n]
|
||||
|
||||
|
||||
def active_searcher(cfg, delegate, *, workspace_id: str | None = None):
|
||||
corpus_root = cfg.paths.artifacts.parent / "corpus"
|
||||
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id)
|
||||
|
||||
|
||||
def validate_corpus_workspace(cfg, workspace_id: str) -> None:
|
||||
"""Fail before downstream retrieval setup when configured corpus ownership differs."""
|
||||
corpus = CorpusStore(cfg.paths.artifacts.parent / "corpus")
|
||||
with corpus.writer_lock():
|
||||
manifest = corpus.active_manifest()
|
||||
if manifest is None:
|
||||
return
|
||||
persisted = manifest.metadata.get("workspace_id")
|
||||
if not isinstance(persisted, str) or re.fullmatch(
|
||||
r"[a-z][a-z0-9_-]{0,63}", persisted
|
||||
) is None:
|
||||
raise CorpusWorkspaceMismatchError(
|
||||
"corpus workspace ownership is missing or invalid; use a new corpus root or rebuild"
|
||||
)
|
||||
if persisted != workspace_id:
|
||||
raise CorpusWorkspaceMismatchError(
|
||||
"corpus belongs to a different workspace; use a new corpus root or rebuild"
|
||||
)
|
||||
|
||||
|
||||
def resolve_citation(
|
||||
store: CorpusStore, evidence_id: str, *, materialized_root=None,
|
||||
) -> str:
|
||||
with store.writer_lock():
|
||||
manifest = store.active_manifest()
|
||||
if manifest is None:
|
||||
return ""
|
||||
for document in manifest.documents:
|
||||
frontmatter = document.metadata.get("frontmatter", {})
|
||||
identifiers = {document.document_id, document.source_id, str(frontmatter.get("id", ""))}
|
||||
if evidence_id in identifiers:
|
||||
root = materialized_root or (store.root / "runtime")
|
||||
filename = document.document_id.removeprefix("doc:") + ".md"
|
||||
path = store.materialize_document(
|
||||
document.document_id, root / filename, generation=manifest.manifest_id,
|
||||
)
|
||||
return str(path) if path else ""
|
||||
return ""
|
||||
|
||||
|
||||
def build_retrieval_entries(results, *, excerpt_chars: int) -> list[dict]:
|
||||
"""Project ordered Evidence search hits into the retrieval-pack shape."""
|
||||
return [
|
||||
{
|
||||
"title": result.label,
|
||||
"status": result.status,
|
||||
"excerpt": result.content[:excerpt_chars],
|
||||
}
|
||||
for result in results
|
||||
]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ActiveEvidenceSearcher",
|
||||
"CorpusWorkspaceMismatchError",
|
||||
"active_searcher",
|
||||
"build_retrieval_entries",
|
||||
"resolve_citation",
|
||||
"validate_corpus_workspace",
|
||||
]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Evidence-specific projection into persisted session artifacts."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
from tht.evidence.search import resolve_citation
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tht.decisions import DecisionRecord
|
||||
from tht.session.models import SchemaLinking
|
||||
|
||||
|
||||
def _resolve_session_citation(evidence_root: Path, evidence_id: str) -> str:
|
||||
# New deployments resolve only immutable materialized files from ACTIVE. Keep the
|
||||
# curated-tree fallback for sessions created before a canonical corpus exists.
|
||||
corpus_root = evidence_root.parent.parent / "corpus"
|
||||
if corpus_root.exists():
|
||||
return resolve_citation(
|
||||
CorpusStore(corpus_root),
|
||||
evidence_id,
|
||||
materialized_root=evidence_root.parent / ".materialized-evidence",
|
||||
)
|
||||
for match in evidence_root.rglob(f"{evidence_id}.md"):
|
||||
return str(match)
|
||||
return ""
|
||||
|
||||
|
||||
def project_session(
|
||||
decisions: list["DecisionRecord"],
|
||||
linking: "SchemaLinking",
|
||||
evidence_root: Path,
|
||||
) -> list[dict]:
|
||||
"""Project cited and reviewed Evidence into the session artifact shape."""
|
||||
entries: dict[str, dict] = {}
|
||||
for candidate in linking.candidates:
|
||||
for evidence_id in candidate.evidence:
|
||||
entries.setdefault(
|
||||
evidence_id,
|
||||
{
|
||||
"id": evidence_id,
|
||||
"file": _resolve_session_citation(evidence_root, evidence_id),
|
||||
"esito": "usata",
|
||||
"decision_seq": candidate.decision_seq,
|
||||
},
|
||||
)
|
||||
for decision in decisions:
|
||||
if decision.type not in ("evidence_accepted", "evidence_rejected"):
|
||||
continue
|
||||
entries[decision.subject] = {
|
||||
"id": decision.subject,
|
||||
"file": _resolve_session_citation(evidence_root, decision.subject),
|
||||
"esito": "accettata" if decision.type == "evidence_accepted" else "scartata",
|
||||
"decision_seq": decision.seq,
|
||||
}
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
__all__ = ["project_session"]
|
||||
@@ -1,10 +1,16 @@
|
||||
"""Construction of configured Evidence source adapters."""
|
||||
|
||||
from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
|
||||
from tht.adapters.evidence.s3 import S3EvidenceSource
|
||||
from tht.config import ConfigError, EvidenceSourcesConfig
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from tht.evidence.adapters import FilesystemEvidenceSource, HttpManifestEvidenceSource
|
||||
from tht.evidence.adapters.s3 import S3EvidenceSource
|
||||
from tht.evidence.contracts import EvidenceSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tht.config import EvidenceSourcesConfig
|
||||
|
||||
|
||||
def build_sources(evidence: EvidenceSourcesConfig | None) -> list[EvidenceSource]:
|
||||
"""Build configured Evidence adapters in the existing deterministic order."""
|
||||
@@ -59,6 +65,8 @@ def build_sources(evidence: EvidenceSourcesConfig | None) -> list[EvidenceSource
|
||||
)
|
||||
)
|
||||
case other: # pragma: no cover - Pydantic rejects unsupported discriminators.
|
||||
from tht.config import ConfigError
|
||||
|
||||
raise ConfigError(f"Adapter evidence non supportato: {other}")
|
||||
return sources
|
||||
|
||||
|
||||
Reference in New Issue
Block a user