refactor(evidence): migrate acquisition and preprocessing (#30)
This commit is contained in:
@@ -7,33 +7,69 @@ and exceptions; they do not define a cross-domain service protocol.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from tht.ports.evidence import (
|
||||
from tht.evidence.acquisition import acquire, discover
|
||||
from tht.evidence.contracts import (
|
||||
AcquiredDocument,
|
||||
EvidenceSource,
|
||||
EvidenceSourceError,
|
||||
EvidenceSourceErrorCategory,
|
||||
SourceObject,
|
||||
canonical_provenance_uri,
|
||||
normalize_aware_datetime,
|
||||
validate_namespaced_value,
|
||||
validate_safe_metadata,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tht.config import EvidenceSourcesConfig
|
||||
from tht.corpus.chunk import ChunkPolicy
|
||||
from tht.corpus.pipeline import CorpusPipeline
|
||||
from tht.corpus.store import CorpusStore
|
||||
from tht.decisions import DecisionRecord
|
||||
from tht.evidence.preprocessing import EvidenceEmbedder
|
||||
from tht.ports.vector import VectorStore
|
||||
from tht.search.evidence import ActiveEvidenceSearcher
|
||||
from tht.session.models import SchemaLinking
|
||||
|
||||
|
||||
def discover(source: EvidenceSource) -> Iterable[SourceObject]:
|
||||
"""Discover source objects without changing source-defined laziness or ordering."""
|
||||
return source.discover()
|
||||
def build_sources(evidence: "EvidenceSourcesConfig | None") -> list[EvidenceSource]:
|
||||
"""Build configured Evidence source adapters in the existing deterministic order."""
|
||||
from tht.evidence.sources import build_sources as build_configured_sources
|
||||
|
||||
return build_configured_sources(evidence)
|
||||
|
||||
|
||||
def acquire(source: EvidenceSource, item: SourceObject) -> AcquiredDocument:
|
||||
"""Acquire one discovered object, preserving the source's classified failures."""
|
||||
return source.acquire(item)
|
||||
def build_preprocessing_pipeline(
|
||||
*,
|
||||
store: "CorpusStore",
|
||||
sources: list[EvidenceSource],
|
||||
embedder: "EvidenceEmbedder",
|
||||
vector_store: "VectorStore",
|
||||
embedding_model: str,
|
||||
embedding_dimensions: int,
|
||||
chunk_policy: "ChunkPolicy",
|
||||
pipeline_version: str,
|
||||
retain_published_generations: int = 3,
|
||||
workspace_id: str | None = None,
|
||||
) -> "CorpusPipeline":
|
||||
"""Construct the Evidence preprocessing use case from core-owned infrastructure."""
|
||||
from tht.evidence.preprocessing import build_preprocessing_pipeline as build_pipeline
|
||||
|
||||
return build_pipeline(
|
||||
store=store,
|
||||
sources=sources,
|
||||
embedder=embedder,
|
||||
vector_store=vector_store,
|
||||
embedding_model=embedding_model,
|
||||
embedding_dimensions=embedding_dimensions,
|
||||
chunk_policy=chunk_policy,
|
||||
pipeline_version=pipeline_version,
|
||||
retain_published_generations=retain_published_generations,
|
||||
workspace_id=workspace_id,
|
||||
)
|
||||
|
||||
|
||||
def active_searcher(
|
||||
@@ -135,9 +171,15 @@ __all__ = [
|
||||
"SourceObject",
|
||||
"acquire",
|
||||
"active_searcher",
|
||||
"build_preprocessing_pipeline",
|
||||
"build_retrieval_entries",
|
||||
"build_sources",
|
||||
"canonical_provenance_uri",
|
||||
"discover",
|
||||
"normalize_aware_datetime",
|
||||
"project_session",
|
||||
"resolve_citation",
|
||||
"validate_corpus_workspace",
|
||||
"validate_namespaced_value",
|
||||
"validate_safe_metadata",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Evidence acquisition operations over the credential-free source contract."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from tht.evidence.contracts import AcquiredDocument, EvidenceSource, SourceObject
|
||||
|
||||
|
||||
def discover(source: EvidenceSource) -> Iterable[SourceObject]:
|
||||
"""Discover source objects without changing source-defined laziness or ordering."""
|
||||
return source.discover()
|
||||
|
||||
|
||||
def acquire(source: EvidenceSource, item: SourceObject) -> AcquiredDocument:
|
||||
"""Acquire one discovered object, preserving the source's classified failures."""
|
||||
return source.acquire(item)
|
||||
|
||||
|
||||
__all__ = ["acquire", "discover"]
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Credential-free contracts for discovering and acquiring Evidence objects."""
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from typing import Protocol, Self, runtime_checkable
|
||||
from urllib.parse import parse_qsl, urlsplit, urlunsplit
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator
|
||||
|
||||
|
||||
class FrozenDict(dict):
|
||||
"""A JSON-serializable dict whose mutation operations are disabled."""
|
||||
|
||||
def _immutable(self, *args, **kwargs):
|
||||
raise TypeError("frozen JSON metadata cannot be mutated")
|
||||
|
||||
__delitem__ = _immutable
|
||||
__ior__ = _immutable
|
||||
__setitem__ = _immutable
|
||||
clear = _immutable
|
||||
pop = _immutable
|
||||
popitem = _immutable
|
||||
setdefault = _immutable
|
||||
update = _immutable
|
||||
|
||||
|
||||
class FrozenList(list):
|
||||
"""A JSON-serializable list whose mutation operations are disabled."""
|
||||
|
||||
def _immutable(self, *args, **kwargs):
|
||||
raise TypeError("frozen JSON metadata cannot be mutated")
|
||||
|
||||
__delitem__ = _immutable
|
||||
__iadd__ = _immutable
|
||||
__imul__ = _immutable
|
||||
__setitem__ = _immutable
|
||||
append = _immutable
|
||||
clear = _immutable
|
||||
extend = _immutable
|
||||
insert = _immutable
|
||||
pop = _immutable
|
||||
remove = _immutable
|
||||
reverse = _immutable
|
||||
sort = _immutable
|
||||
|
||||
|
||||
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
|
||||
_SEPARATORS = re.compile(r"[^a-z0-9]+")
|
||||
_NAMESPACED_VALUE = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
||||
_CREDENTIAL_KEYS = {
|
||||
"apikey",
|
||||
"authorization",
|
||||
"authtoken",
|
||||
"bearertoken",
|
||||
"clientsecret",
|
||||
"credential",
|
||||
"credentials",
|
||||
"password",
|
||||
"passwd",
|
||||
"privatekey",
|
||||
"refreshtoken",
|
||||
"sessioncookie",
|
||||
"xapikey",
|
||||
"accesstoken",
|
||||
}
|
||||
_JSON_METADATA = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _normalize_key(key: str) -> str:
|
||||
return _SEPARATORS.sub("", _CAMEL_BOUNDARY.sub("_", key).lower())
|
||||
|
||||
|
||||
def _is_credential_key(key: str) -> bool:
|
||||
return _normalize_key(key) in _CREDENTIAL_KEYS
|
||||
|
||||
|
||||
def _reject_credentials(value, path: str = "metadata") -> None:
|
||||
if isinstance(value, Mapping):
|
||||
for key, child in value.items():
|
||||
if _is_credential_key(str(key)):
|
||||
raise ValueError(f"credential-like metadata key is not allowed: {path}.{key}")
|
||||
_reject_credentials(child, f"{path}.{key}")
|
||||
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
for index, child in enumerate(value):
|
||||
_reject_credentials(child, f"{path}[{index}]")
|
||||
|
||||
|
||||
def freeze_json(value):
|
||||
"""Recursively freeze a Pydantic-validated JSON value without changing its JSON shape."""
|
||||
if isinstance(value, Mapping):
|
||||
return FrozenDict({str(key): freeze_json(child) for key, child in value.items()})
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return FrozenList(freeze_json(child) for child in value)
|
||||
return value
|
||||
|
||||
|
||||
def validate_safe_metadata(value: dict[str, JsonValue]) -> FrozenDict:
|
||||
_reject_credentials(value)
|
||||
return freeze_json(value)
|
||||
|
||||
|
||||
def validate_canonical_uri(value: str) -> str:
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
_ = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValueError("invalid canonical URI") from error
|
||||
if not parsed.scheme:
|
||||
raise ValueError("canonical URI must include a scheme")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValueError("canonical URI must not contain credentials in userinfo")
|
||||
for key, _ in parse_qsl(parsed.query, keep_blank_values=True):
|
||||
if _is_credential_key(key):
|
||||
raise ValueError("canonical URI must not contain credentials in query parameters")
|
||||
return value
|
||||
|
||||
|
||||
def canonical_provenance_uri(value: str) -> str:
|
||||
"""Return only stable URI identity; transport query/fragment data is never provenance."""
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
_ = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValueError("invalid canonical URI") from error
|
||||
if not parsed.scheme:
|
||||
raise ValueError("canonical URI must include a scheme")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValueError("canonical URI must not contain credentials in userinfo")
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
|
||||
|
||||
|
||||
def normalize_aware_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("datetime must be timezone-aware")
|
||||
return value.astimezone(UTC)
|
||||
|
||||
|
||||
def validate_namespaced_value(value: str) -> str:
|
||||
if not _NAMESPACED_VALUE.fullmatch(value):
|
||||
raise ValueError("value must be namespaced as '<kind>:<stable-value>'")
|
||||
return value
|
||||
|
||||
|
||||
class _EvidenceValue(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
frozen=True,
|
||||
extra="forbid",
|
||||
revalidate_instances="always",
|
||||
validate_default=True,
|
||||
ser_json_bytes="base64",
|
||||
val_json_bytes="base64",
|
||||
)
|
||||
|
||||
def model_copy(self, *, update: Mapping[str, object] | None = None, deep: bool = False) -> Self:
|
||||
"""Copy through validation; Pydantic's unchecked update-copy is unsafe for contracts."""
|
||||
data = self.model_dump(round_trip=True)
|
||||
if update:
|
||||
data.update(update)
|
||||
return type(self).model_validate(data)
|
||||
|
||||
|
||||
class SourceObject(_EvidenceValue):
|
||||
source_id: str = Field(min_length=1)
|
||||
uri: str = Field(min_length=1)
|
||||
fingerprint: str = Field(min_length=1)
|
||||
modified_at: datetime | None = None
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
_source_id = field_validator("source_id")(validate_namespaced_value)
|
||||
_fingerprint = field_validator("fingerprint")(validate_namespaced_value)
|
||||
_safe_uri = field_validator("uri")(validate_canonical_uri)
|
||||
_aware_modified_at = field_validator("modified_at")(normalize_aware_datetime)
|
||||
_frozen_metadata = field_validator("metadata")(validate_safe_metadata)
|
||||
|
||||
|
||||
class AcquiredDocument(_EvidenceValue):
|
||||
"""Transport result; bytes use explicit base64 encoding in JSON mode."""
|
||||
|
||||
source: SourceObject
|
||||
content: bytes
|
||||
media_type: str | None = None
|
||||
acquired_at: datetime | None = None
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
_aware_acquired_at = field_validator("acquired_at")(normalize_aware_datetime)
|
||||
_frozen_metadata = field_validator("metadata")(validate_safe_metadata)
|
||||
|
||||
|
||||
class EvidenceSourceErrorCategory(str, Enum):
|
||||
TRANSIENT = "transient"
|
||||
PERMANENT = "permanent"
|
||||
|
||||
|
||||
class EvidenceSourceError(Exception):
|
||||
"""Classified source failure with credential-free structured diagnostics."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
_message: str,
|
||||
*,
|
||||
category: EvidenceSourceErrorCategory,
|
||||
details: dict[str, JsonValue] | None = None,
|
||||
) -> None:
|
||||
super().__init__("evidence source operation failed")
|
||||
object.__setattr__(self, "category", EvidenceSourceErrorCategory(category))
|
||||
object.__setattr__(
|
||||
self,
|
||||
"details",
|
||||
validate_safe_metadata(_JSON_METADATA.validate_python(details or {})),
|
||||
)
|
||||
|
||||
def __setattr__(self, name: str, value) -> None:
|
||||
if name in {"args", "category", "details"} and hasattr(self, name):
|
||||
raise AttributeError(f"{name} is immutable")
|
||||
super().__setattr__(name, value)
|
||||
|
||||
@property
|
||||
def retryable(self) -> bool:
|
||||
return self.category is EvidenceSourceErrorCategory.TRANSIENT
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EvidenceSource(Protocol):
|
||||
def discover(self) -> Iterable[SourceObject]: ...
|
||||
|
||||
def acquire(self, item: SourceObject) -> AcquiredDocument: ...
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Explicit construction boundary for Evidence preprocessing."""
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from tht.corpus.chunk import ChunkPolicy
|
||||
from tht.corpus.pipeline import CorpusPipeline
|
||||
from tht.corpus.store import CorpusStore
|
||||
from tht.evidence.contracts import EvidenceSource
|
||||
from tht.ports.vector import VectorStore
|
||||
|
||||
|
||||
class EvidenceEmbedder(Protocol):
|
||||
def embed_documents(self, texts: list[str]) -> list[list[float]]: ...
|
||||
|
||||
|
||||
def build_preprocessing_pipeline(
|
||||
*,
|
||||
store: CorpusStore,
|
||||
sources: list[EvidenceSource],
|
||||
embedder: EvidenceEmbedder,
|
||||
vector_store: VectorStore,
|
||||
embedding_model: str,
|
||||
embedding_dimensions: int,
|
||||
chunk_policy: ChunkPolicy,
|
||||
pipeline_version: str,
|
||||
retain_published_generations: int = 3,
|
||||
workspace_id: str | None = None,
|
||||
) -> CorpusPipeline:
|
||||
"""Construct preprocessing from the bounded infrastructure supplied by core."""
|
||||
return CorpusPipeline(
|
||||
store=store,
|
||||
sources=sources,
|
||||
embedder=embedder,
|
||||
vector_store=vector_store,
|
||||
embedding_model=embedding_model,
|
||||
embedding_dimensions=embedding_dimensions,
|
||||
chunk_policy=chunk_policy,
|
||||
pipeline_version=pipeline_version,
|
||||
retain_published_generations=retain_published_generations,
|
||||
workspace_id=workspace_id,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["EvidenceEmbedder", "build_preprocessing_pipeline"]
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Construction of configured Evidence source adapters."""
|
||||
|
||||
from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
|
||||
from tht.adapters.evidence.s3 import S3EvidenceSource
|
||||
from tht.config import ConfigError, EvidenceSourcesConfig
|
||||
from tht.evidence.contracts import EvidenceSource
|
||||
|
||||
|
||||
def build_sources(evidence: EvidenceSourcesConfig | None) -> list[EvidenceSource]:
|
||||
"""Build configured Evidence adapters in the existing deterministic order."""
|
||||
if evidence is None:
|
||||
return []
|
||||
sources: list[EvidenceSource] = []
|
||||
if evidence.source_root is not None:
|
||||
sources.append(FilesystemEvidenceSource(evidence.source_root / evidence.evidence_dir))
|
||||
for resource in evidence.sources:
|
||||
match resource.type:
|
||||
case "filesystem":
|
||||
sources.append(
|
||||
FilesystemEvidenceSource(
|
||||
resource.root,
|
||||
patterns=resource.patterns,
|
||||
max_bytes=resource.max_bytes,
|
||||
)
|
||||
)
|
||||
case "http":
|
||||
sources.append(
|
||||
HttpManifestEvidenceSource(
|
||||
resource.transport_urls(),
|
||||
connect_timeout=resource.connect_timeout,
|
||||
read_timeout=resource.read_timeout,
|
||||
max_bytes=resource.max_bytes,
|
||||
max_redirects=resource.max_redirects,
|
||||
allow_private_hosts=resource.allow_private_hosts,
|
||||
max_cache_bytes=resource.max_cache_bytes,
|
||||
)
|
||||
)
|
||||
case "s3":
|
||||
|
||||
def secret(value):
|
||||
return value.get_secret_value() if value is not None else None
|
||||
|
||||
sources.append(
|
||||
S3EvidenceSource(
|
||||
bucket=resource.bucket,
|
||||
prefix=resource.prefix,
|
||||
endpoint_url=resource.endpoint_url,
|
||||
region=resource.region,
|
||||
access_key=secret(resource.access_key),
|
||||
secret_key=secret(resource.secret_key),
|
||||
session_token=secret(resource.session_token),
|
||||
trusted_endpoint=resource.trusted_endpoint,
|
||||
allow_private_endpoint=resource.allow_private_endpoint,
|
||||
allow_insecure_endpoint=resource.allow_insecure_endpoint,
|
||||
max_bytes=resource.max_bytes,
|
||||
max_objects=resource.max_objects,
|
||||
max_pages=resource.max_pages,
|
||||
page_size=resource.page_size,
|
||||
)
|
||||
)
|
||||
case other: # pragma: no cover - Pydantic rejects unsupported discriminators.
|
||||
raise ConfigError(f"Adapter evidence non supportato: {other}")
|
||||
return sources
|
||||
|
||||
|
||||
__all__ = ["build_sources"]
|
||||
Reference in New Issue
Block a user