refactor(evidence): remove legacy Python layout (#31)
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Evidence-owned canonical, transport-independent corpus."""
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Evidence-owned deterministic chunking for canonical corpus documents."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChunkPolicy:
|
||||
version: str
|
||||
max_chars: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.version:
|
||||
raise ValueError("chunk policy version must not be empty")
|
||||
if self.max_chars <= 0:
|
||||
raise ValueError("max_chars must be greater than zero")
|
||||
|
||||
|
||||
def _hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _contents(content: str, maximum: int) -> list[str]:
|
||||
result: list[str] = []
|
||||
start = 0
|
||||
while start < len(content):
|
||||
end = min(start + maximum, len(content))
|
||||
if end < len(content):
|
||||
boundaries = list(re.finditer(r"\s+", content[start:end]))
|
||||
if boundaries:
|
||||
end = start + boundaries[-1].end()
|
||||
result.append(content[start:end])
|
||||
start = end
|
||||
return result
|
||||
|
||||
|
||||
def _policy_fingerprint(policy: ChunkPolicy) -> str:
|
||||
serialized = json.dumps(asdict(policy), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return f"sha256:{_hash(serialized)}"
|
||||
|
||||
|
||||
def chunk(document: CanonicalDocument, policy: ChunkPolicy) -> list[CanonicalChunk]:
|
||||
"""Split canonical text with stable character-count boundaries and identifiers."""
|
||||
chunks: list[CanonicalChunk] = []
|
||||
policy_fingerprint = _policy_fingerprint(policy)
|
||||
for ordinal, content in enumerate(_contents(document.content, policy.max_chars)):
|
||||
chunk_hash = f"sha256:{_hash(content)}"
|
||||
identifier = _hash(
|
||||
":".join(
|
||||
(
|
||||
document.document_id,
|
||||
document.content_hash,
|
||||
policy_fingerprint,
|
||||
str(ordinal),
|
||||
chunk_hash,
|
||||
)
|
||||
)
|
||||
)
|
||||
chunks.append(
|
||||
CanonicalChunk(
|
||||
chunk_id=f"chunk:{identifier}",
|
||||
document_id=document.document_id,
|
||||
ordinal=ordinal,
|
||||
content=content,
|
||||
content_hash=chunk_hash,
|
||||
source_uri=document.source_uri,
|
||||
pipeline_version=document.pipeline_version,
|
||||
metadata={
|
||||
"chunk_policy": {
|
||||
"version": policy.version,
|
||||
"max_chars": policy.max_chars,
|
||||
"fingerprint": policy_fingerprint,
|
||||
},
|
||||
"document": document.model_dump(mode="json")["metadata"],
|
||||
"source_fingerprint": document.source_fingerprint,
|
||||
"title": document.title,
|
||||
},
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Evidence-owned immutable records emitted by preprocessing."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Self
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
||||
|
||||
from tht.evidence.contracts import (
|
||||
canonical_provenance_uri,
|
||||
normalize_aware_datetime,
|
||||
validate_namespaced_value,
|
||||
validate_safe_metadata,
|
||||
)
|
||||
|
||||
|
||||
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
||||
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def _validate_namespaced_id(value: str) -> str:
|
||||
if not _NAMESPACED_ID.fullmatch(value):
|
||||
raise ValueError("identifier must be namespaced as '<kind>:<stable-value>'")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_hash(value: str) -> str:
|
||||
if not _SHA256.fullmatch(value):
|
||||
raise ValueError("content hash must be 'sha256:' followed by 64 lowercase hex digits")
|
||||
return value
|
||||
|
||||
|
||||
def _require_content_hash(content: str, content_hash: str) -> None:
|
||||
expected = f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
|
||||
if content_hash != expected:
|
||||
raise ValueError("content_hash must match the exact canonical UTF-8 content")
|
||||
|
||||
|
||||
class _CanonicalValue(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
frozen=True, extra="forbid", validate_default=True, revalidate_instances="always"
|
||||
)
|
||||
|
||||
def model_copy(self, *, update: Mapping[str, object] | None = None, deep: bool = False) -> Self:
|
||||
"""Copy through full field and model validation, including manifest invariants."""
|
||||
data = self.model_dump(round_trip=True)
|
||||
if update:
|
||||
data.update(update)
|
||||
return type(self).model_validate(data)
|
||||
|
||||
|
||||
class _WithMetadata(_CanonicalValue):
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
_frozen_metadata = field_validator("metadata")(validate_safe_metadata)
|
||||
|
||||
|
||||
class CanonicalDocument(_WithMetadata):
|
||||
"""Normalized text whose hash covers the exact stored UTF-8 content bytes."""
|
||||
document_id: str
|
||||
source_id: str
|
||||
source_uri: str
|
||||
source_fingerprint: str = Field(min_length=1)
|
||||
content_hash: str
|
||||
title: str = ""
|
||||
content: str
|
||||
media_type: str = "text/plain"
|
||||
modified_at: datetime | None = None
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
_document_id = field_validator("document_id")(_validate_namespaced_id)
|
||||
_source_id = field_validator("source_id")(_validate_namespaced_id)
|
||||
_source_uri = field_validator("source_uri")(canonical_provenance_uri)
|
||||
_source_fingerprint = field_validator("source_fingerprint")(validate_namespaced_value)
|
||||
_content_hash = field_validator("content_hash")(_validate_hash)
|
||||
_modified_at = field_validator("modified_at")(normalize_aware_datetime)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content_hash_matches(self) -> "CanonicalDocument":
|
||||
_require_content_hash(self.content, self.content_hash)
|
||||
return self
|
||||
|
||||
|
||||
class CanonicalChunk(_WithMetadata):
|
||||
"""Chunk text whose hash covers the exact stored UTF-8 content bytes."""
|
||||
chunk_id: str
|
||||
document_id: str
|
||||
ordinal: int = Field(ge=0)
|
||||
content: str
|
||||
content_hash: str
|
||||
source_uri: str
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
_chunk_id = field_validator("chunk_id")(_validate_namespaced_id)
|
||||
_document_id = field_validator("document_id")(_validate_namespaced_id)
|
||||
_content_hash = field_validator("content_hash")(_validate_hash)
|
||||
_source_uri = field_validator("source_uri")(canonical_provenance_uri)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content_hash_matches(self) -> "CanonicalChunk":
|
||||
_require_content_hash(self.content, self.content_hash)
|
||||
return self
|
||||
|
||||
|
||||
class CorpusManifest(_WithMetadata):
|
||||
"""Description of one internally consistent publishable generation."""
|
||||
|
||||
schema_version: int = Field(default=1, ge=1)
|
||||
manifest_id: str | None = None
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
pipeline_version: str = Field(default="evidence-v1", min_length=1)
|
||||
embedding_model: str | None = None
|
||||
embedding_dimensions: int | None = Field(default=None, gt=0)
|
||||
vector_generation: str | None = None
|
||||
documents: tuple[CanonicalDocument, ...] = Field(default_factory=tuple)
|
||||
chunks: tuple[CanonicalChunk, ...] = Field(default_factory=tuple)
|
||||
|
||||
_manifest_id = field_validator("manifest_id")(
|
||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||
)
|
||||
_vector_generation = field_validator("vector_generation")(
|
||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||
)
|
||||
_created_at = field_validator("created_at")(normalize_aware_datetime)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_generation(self) -> "CorpusManifest":
|
||||
if (self.embedding_model is None) != (self.embedding_dimensions is None):
|
||||
raise ValueError("embedding_model and embedding_dimensions must be set together")
|
||||
if self.vector_generation is not None and self.embedding_model is None:
|
||||
raise ValueError("vector_generation requires embedding model and dimension compatibility")
|
||||
|
||||
document_ids = [document.document_id for document in self.documents]
|
||||
source_ids = [document.source_id for document in self.documents]
|
||||
chunk_ids = [chunk.chunk_id for chunk in self.chunks]
|
||||
self._require_unique("document_id", document_ids)
|
||||
self._require_unique("source_id", source_ids)
|
||||
self._require_unique("chunk_id", chunk_ids)
|
||||
|
||||
documents = {document.document_id: document for document in self.documents}
|
||||
ordinals: dict[str, list[int]] = {}
|
||||
for document in self.documents:
|
||||
if document.pipeline_version != self.pipeline_version:
|
||||
raise ValueError("document pipeline_version must match manifest pipeline_version")
|
||||
for chunk in self.chunks:
|
||||
document = documents.get(chunk.document_id)
|
||||
if document is None:
|
||||
raise ValueError(f"chunk references unknown document: {chunk.document_id}")
|
||||
if chunk.pipeline_version != self.pipeline_version:
|
||||
raise ValueError("chunk pipeline_version must match manifest pipeline_version")
|
||||
if chunk.source_uri != document.source_uri:
|
||||
raise ValueError("chunk source_uri must match its document provenance")
|
||||
ordinals.setdefault(chunk.document_id, []).append(chunk.ordinal)
|
||||
for document_id, values in ordinals.items():
|
||||
if sorted(values) != list(range(len(values))):
|
||||
raise ValueError(f"chunk ordinals must be unique and contiguous for {document_id}")
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _require_unique(field: str, values: list[str]) -> None:
|
||||
if len(values) != len(set(values)):
|
||||
raise ValueError(f"{field} values must be unique")
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Evidence-owned deterministic conversion of acquired bytes into canonical text."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Mapping
|
||||
|
||||
import yaml
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from yaml.events import AliasEvent
|
||||
from yaml.nodes import MappingNode
|
||||
|
||||
from tht.evidence.corpus.models import CanonicalDocument
|
||||
from tht.evidence.contracts import AcquiredDocument, canonical_provenance_uri
|
||||
|
||||
|
||||
MAX_DOCUMENT_BYTES = 10 * 1024 * 1024
|
||||
_CHARSET = re.compile(r"(?:^|;)\s*charset\s*=\s*[\"']?([^;\s\"']+)", re.IGNORECASE)
|
||||
_FRONTMATTER = re.compile(r"\A---\n(.*?)\n---(?:\n|\Z)", re.DOTALL)
|
||||
_JSON_OBJECT = TypeAdapter(dict[str, JsonValue])
|
||||
_MAX_FRONTMATTER_DEPTH = 20
|
||||
_MAX_FRONTMATTER_NODES = 1000
|
||||
|
||||
|
||||
class _FrontmatterLoader(yaml.SafeLoader):
|
||||
"""SafeLoader with bounded structure and no YAML graph features."""
|
||||
|
||||
def __init__(self, stream) -> None:
|
||||
super().__init__(stream)
|
||||
self._depth = 0
|
||||
self._nodes = 0
|
||||
|
||||
def compose_node(self, parent, index):
|
||||
event = self.peek_event()
|
||||
if isinstance(event, AliasEvent) or getattr(event, "anchor", None) is not None:
|
||||
raise yaml.constructor.ConstructorError(None, None, "aliases are not allowed")
|
||||
self._depth += 1
|
||||
self._nodes += 1
|
||||
if self._depth > _MAX_FRONTMATTER_DEPTH or self._nodes > _MAX_FRONTMATTER_NODES:
|
||||
raise yaml.constructor.ConstructorError(None, None, "frontmatter is too complex")
|
||||
try:
|
||||
return super().compose_node(parent, index)
|
||||
finally:
|
||||
self._depth -= 1
|
||||
|
||||
def construct_mapping(self, node, deep=False):
|
||||
if not isinstance(node, MappingNode):
|
||||
return super().construct_mapping(node, deep=deep)
|
||||
seen: set[object] = set()
|
||||
for key_node, _ in node.value:
|
||||
key = self.construct_object(key_node, deep=deep)
|
||||
try:
|
||||
duplicate = key in seen
|
||||
seen.add(key)
|
||||
except TypeError as error:
|
||||
raise yaml.constructor.ConstructorError(
|
||||
None, None, "mapping keys must be scalar"
|
||||
) from error
|
||||
if duplicate:
|
||||
raise yaml.constructor.ConstructorError(None, None, "duplicate mapping key")
|
||||
return super().construct_mapping(node, deep=deep)
|
||||
|
||||
|
||||
class PermanentNormalizationError(ValueError):
|
||||
"""A deterministic input failure which retrying cannot repair."""
|
||||
|
||||
def __init__(self, reason: str) -> None:
|
||||
super().__init__(f"document normalization failed: {reason}")
|
||||
self.reason = reason
|
||||
self.permanent = True
|
||||
|
||||
|
||||
def _sha256(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _decode(acquired: AcquiredDocument) -> str:
|
||||
if len(acquired.content) > MAX_DOCUMENT_BYTES:
|
||||
raise PermanentNormalizationError("oversized")
|
||||
|
||||
media_type = acquired.media_type or "text/plain"
|
||||
charset = _CHARSET.search(media_type)
|
||||
if charset and charset.group(1).lower().replace("_", "-") not in {
|
||||
"utf-8",
|
||||
"utf8",
|
||||
"us-ascii",
|
||||
"ascii",
|
||||
}:
|
||||
raise PermanentNormalizationError("unsupported_charset")
|
||||
try:
|
||||
return acquired.content.decode("utf-8-sig", errors="strict")
|
||||
except UnicodeDecodeError as error:
|
||||
raise PermanentNormalizationError("undecodable") from error
|
||||
|
||||
|
||||
def _frontmatter(text: str) -> tuple[dict[str, JsonValue], str]:
|
||||
match = _FRONTMATTER.match(text)
|
||||
if match is None:
|
||||
return {}, text
|
||||
try:
|
||||
loaded = yaml.load(match.group(1), Loader=_FrontmatterLoader)
|
||||
if loaded is None:
|
||||
loaded = {}
|
||||
if not isinstance(loaded, Mapping):
|
||||
raise TypeError("frontmatter is not a mapping")
|
||||
metadata = _JSON_OBJECT.validate_python(dict(loaded))
|
||||
except (TypeError, UnicodeError, ValidationError, yaml.YAMLError) as error:
|
||||
raise PermanentNormalizationError("invalid_frontmatter") from error
|
||||
return metadata, text[match.end() :]
|
||||
|
||||
|
||||
def normalize(acquired: AcquiredDocument, pipeline_version: str) -> CanonicalDocument:
|
||||
"""Normalize one transport result without I/O or implicit data loss."""
|
||||
if not pipeline_version:
|
||||
raise ValueError("pipeline_version must not be empty")
|
||||
|
||||
decoded = _decode(acquired)
|
||||
canonical = unicodedata.normalize("NFC", decoded.replace("\r\n", "\n").replace("\r", "\n"))
|
||||
frontmatter, content = _frontmatter(canonical)
|
||||
source_uri = canonical_provenance_uri(acquired.source.uri)
|
||||
identity = f"{acquired.source.source_id}\n{source_uri}"
|
||||
media_type = (acquired.media_type or "text/plain").split(";", 1)[0].strip().lower()
|
||||
metadata: dict[str, JsonValue] = {
|
||||
"source": acquired.source.model_dump(mode="json")["metadata"],
|
||||
"acquisition": acquired.model_dump(mode="json")["metadata"],
|
||||
}
|
||||
if frontmatter:
|
||||
metadata["frontmatter"] = frontmatter
|
||||
|
||||
try:
|
||||
return CanonicalDocument(
|
||||
document_id=f"doc:{_sha256(identity)}",
|
||||
source_id=acquired.source.source_id,
|
||||
source_uri=source_uri,
|
||||
source_fingerprint=acquired.source.fingerprint,
|
||||
content_hash=f"sha256:{_sha256(content)}",
|
||||
title=str(frontmatter.get("title", "")),
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
modified_at=acquired.source.modified_at,
|
||||
pipeline_version=pipeline_version,
|
||||
metadata=metadata,
|
||||
)
|
||||
except ValidationError as error:
|
||||
if frontmatter:
|
||||
raise PermanentNormalizationError("invalid_frontmatter") from error
|
||||
raise
|
||||
@@ -0,0 +1,811 @@
|
||||
"""Evidence-owned incremental preprocessing with generation-isolated vector writes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC
|
||||
from pathlib import Path
|
||||
|
||||
from tht.evidence.corpus.chunk import ChunkPolicy, chunk
|
||||
from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
|
||||
from tht.evidence.corpus.normalize import normalize
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
import tht.evidence.acquisition as evidence_acquisition
|
||||
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
from tht.ports.vector import VectorStore, VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
from tht.jobs.models import JobSpec
|
||||
from tht.jobs.runner import JobContext, StageArtifacts, run_job, seal_stage_artifacts
|
||||
|
||||
|
||||
EVIDENCE_STAGE_IDS = (
|
||||
"discover",
|
||||
"acquire_normalize_chunk",
|
||||
"embed",
|
||||
"vector_upsert",
|
||||
"stage_validate",
|
||||
"publish",
|
||||
"retention_cleanup",
|
||||
)
|
||||
|
||||
|
||||
class PipelineError(RuntimeError):
|
||||
"""Credential-free failure at the preprocessing boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PipelineResult:
|
||||
status: str
|
||||
generation: str | None
|
||||
published: bool
|
||||
changed: tuple[str, ...]
|
||||
unchanged: tuple[str, ...]
|
||||
removed: tuple[str, ...]
|
||||
manifest: CorpusManifest = field(repr=False)
|
||||
run_id: str | None = None
|
||||
resumed_from: str | None = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
counts = {
|
||||
"changed": len(self.changed),
|
||||
"unchanged": len(self.unchanged),
|
||||
"removed": len(self.removed),
|
||||
}
|
||||
return (
|
||||
f"PipelineResult(status={self.status!r}, generation={self.generation!r}, "
|
||||
f"published={self.published!r}, counts={counts!r}, "
|
||||
f"run_id={self.run_id!r}, resumed_from={self.resumed_from!r})"
|
||||
)
|
||||
|
||||
def model_dump(self, mode=None):
|
||||
def bounded(values: tuple[str, ...]) -> list[str]:
|
||||
return [value[:200] for value in values[:100]]
|
||||
|
||||
return {
|
||||
"status": self.status,
|
||||
"generation": self.generation,
|
||||
"published": self.published,
|
||||
"changed": bounded(self.changed),
|
||||
"unchanged": bounded(self.unchanged),
|
||||
"removed": bounded(self.removed),
|
||||
"counts": {
|
||||
"changed": len(self.changed),
|
||||
"unchanged": len(self.unchanged),
|
||||
"removed": len(self.removed),
|
||||
"documents": len(self.manifest.documents),
|
||||
"chunks": len(self.manifest.chunks),
|
||||
},
|
||||
"manifest_id": self.manifest.manifest_id,
|
||||
"run_id": self.run_id,
|
||||
"resumed_from": self.resumed_from,
|
||||
}
|
||||
|
||||
|
||||
def _fingerprint(value) -> str:
|
||||
payload = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||
return "sha256:" + hashlib.sha256(payload.encode()).hexdigest()
|
||||
|
||||
|
||||
def _canonical_json(value):
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _canonical_json(value[key]) for key in sorted(value)}
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return [_canonical_json(child) for child in value]
|
||||
return value
|
||||
|
||||
|
||||
def _source_snapshot(discovered) -> dict[str, dict]:
|
||||
snapshot = {}
|
||||
for _, item in discovered:
|
||||
modified_at = item.modified_at.astimezone(UTC) if item.modified_at else None
|
||||
metadata = _canonical_json(item.metadata)
|
||||
snapshot[item.source_id] = {
|
||||
"source_id": item.source_id,
|
||||
"uri": item.uri,
|
||||
"fingerprint": item.fingerprint,
|
||||
"modified_at": modified_at.isoformat().replace("+00:00", "Z") if modified_at else None,
|
||||
"metadata": metadata,
|
||||
"media_type": metadata.get("media_type"),
|
||||
"size": metadata.get("size"),
|
||||
}
|
||||
return snapshot
|
||||
|
||||
|
||||
class CorpusPipeline:
|
||||
def __init__(
|
||||
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
|
||||
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
|
||||
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
|
||||
workspace_id: str | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.sources = sources
|
||||
self.embedder = embedder
|
||||
self.vector_store = vector_store
|
||||
self.embedding_model = embedding_model
|
||||
self.embedding_dimensions = embedding_dimensions
|
||||
self.chunk_policy = chunk_policy
|
||||
self.pipeline_version = pipeline_version
|
||||
if isinstance(retain_published_generations, bool) or retain_published_generations < 1:
|
||||
raise ValueError("retain_published_generations must be at least 1")
|
||||
self.retain_published_generations = retain_published_generations
|
||||
self.workspace_id = workspace_id
|
||||
|
||||
def _assert_workspace_binding(self) -> None:
|
||||
manifest = self.store.active_manifest()
|
||||
if manifest is None:
|
||||
if self.workspace_id is None:
|
||||
self.workspace_id = "default"
|
||||
return
|
||||
persisted = manifest.metadata.get("workspace_id")
|
||||
if not isinstance(persisted, str) or re.fullmatch(
|
||||
r"[a-z][a-z0-9_-]{0,63}", persisted
|
||||
) is None:
|
||||
raise PipelineError(
|
||||
"corpus workspace ownership is missing or invalid; use a new corpus root or explicit rebuild"
|
||||
)
|
||||
if self.workspace_id is None and isinstance(persisted, str):
|
||||
self.workspace_id = persisted
|
||||
return
|
||||
if persisted != self.workspace_id:
|
||||
raise PipelineError(
|
||||
"corpus belongs to a different workspace; use a new corpus root or explicit rebuild"
|
||||
)
|
||||
|
||||
def _protected_generations(self, workspace_root: Path) -> set[str]:
|
||||
protected = {value for value in (self.store.active_generation(),) if value}
|
||||
runs = workspace_root / ".tht-jobs" / "evidence" / "runs"
|
||||
for checkpoint in runs.glob("*/checkpoint.json") if runs.exists() else ():
|
||||
try:
|
||||
state = json.loads(checkpoint.read_text(encoding="utf-8"))
|
||||
if state.get("status") not in {"running", "failed"}:
|
||||
continue
|
||||
plan = checkpoint.parent / "artifacts" / "plan.json"
|
||||
generation = json.loads(plan.read_text(encoding="utf-8")).get("generation")
|
||||
if isinstance(generation, str):
|
||||
protected.add(generation)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return protected
|
||||
|
||||
def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
|
||||
with self.store.writer_lock():
|
||||
return self._gc(workspace_root=workspace_root, dry_run=dry_run)
|
||||
|
||||
def _gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
|
||||
self._assert_workspace_binding()
|
||||
published = self.store.published_generations()
|
||||
list_vectors = getattr(self.vector_store, "list_evidence_generations", None)
|
||||
vector_generations = set(list_vectors("evidence", self.workspace_id)) if list_vectors else set()
|
||||
generations = sorted(set(published) | vector_generations)
|
||||
job_protected = self._protected_generations(workspace_root)
|
||||
active = self.store.active_generation()
|
||||
rollback_count = self.retain_published_generations - 1
|
||||
rollback = [generation for generation in published if generation != active]
|
||||
keep = ({active} if active else set()) | set(rollback[-rollback_count:] if rollback_count else ())
|
||||
fs_keep = keep | job_protected
|
||||
vector_protected = set(fs_keep)
|
||||
for generation in fs_keep:
|
||||
try:
|
||||
manifest = self.store.manifest(generation)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
vector_protected.update(
|
||||
value for value in manifest.metadata.get("document_generations", {}).values()
|
||||
if isinstance(value, str)
|
||||
)
|
||||
evicted, failures = [], []
|
||||
filesystem_generations = set(self.store.list_generations())
|
||||
for generation in generations:
|
||||
purge_vector = generation not in vector_protected
|
||||
purge_filesystem = generation in filesystem_generations and generation not in fs_keep
|
||||
if not purge_vector and not purge_filesystem:
|
||||
continue
|
||||
if dry_run:
|
||||
evicted.append(generation)
|
||||
continue
|
||||
if purge_vector:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
failures.append({"generation": generation, "error": "vector cleanup failed"})
|
||||
continue
|
||||
try:
|
||||
if purge_filesystem:
|
||||
self.store.discard(generation)
|
||||
evicted.append(generation)
|
||||
except Exception:
|
||||
failures.append({"generation": generation, "error": "filesystem cleanup failed"})
|
||||
return {"status": "partial" if failures else "succeeded", "dry_run": dry_run,
|
||||
"active_generation": self.store.active_generation(), "evicted": evicted,
|
||||
"protected": sorted(vector_protected), "failures": failures}
|
||||
|
||||
def _discover(self) -> list[tuple[EvidenceSource, SourceObject]]:
|
||||
discovered = []
|
||||
seen = set()
|
||||
for source in self.sources:
|
||||
for item in evidence_acquisition.discover(source):
|
||||
if item.source_id in seen:
|
||||
raise PipelineError("duplicate Evidence source identity")
|
||||
seen.add(item.source_id)
|
||||
discovered.append((source, item))
|
||||
return sorted(discovered, key=lambda pair: pair[1].source_id)
|
||||
|
||||
def run(self, *, dry_run: bool = False, resume: str | None = None) -> PipelineResult:
|
||||
with self.store.writer_lock():
|
||||
self._assert_workspace_binding()
|
||||
return self._run(dry_run=dry_run, resume=resume)
|
||||
|
||||
def run_as_job(self, **kwargs) -> PipelineResult:
|
||||
self.workspace_id = kwargs["workspace_id"]
|
||||
with self.store.writer_lock():
|
||||
self._assert_workspace_binding()
|
||||
return self._run_as_job(**kwargs)
|
||||
|
||||
def _run_as_job(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
workspace_root: Path,
|
||||
config_fingerprint: str,
|
||||
input_fingerprint: str,
|
||||
dry_run: bool = False,
|
||||
resume_run_id: str | None = None,
|
||||
after_stage_return=None,
|
||||
) -> PipelineResult:
|
||||
"""Execute preprocessing through the durable shared job envelope."""
|
||||
discovered = self._discover()
|
||||
discovered_fingerprint = _fingerprint(
|
||||
{item.source_id: item.fingerprint for _, item in discovered}
|
||||
)
|
||||
source_snapshot = _source_snapshot(discovered)
|
||||
source_by_id = {item.source_id: (source, item) for source, item in discovered}
|
||||
compatibility = _fingerprint({
|
||||
"pipeline": self.pipeline_version,
|
||||
"model": self.embedding_model,
|
||||
"dimensions": self.embedding_dimensions,
|
||||
"chunk_policy": asdict(self.chunk_policy),
|
||||
})
|
||||
job_binding = {
|
||||
"config_fingerprint": config_fingerprint,
|
||||
"input_fingerprint": input_fingerprint,
|
||||
"compatibility_fingerprint": compatibility,
|
||||
"pipeline_version": self.pipeline_version,
|
||||
"chunk_policy_version": self.chunk_policy.version,
|
||||
"embedding_model": self.embedding_model,
|
||||
"embedding_dimensions": self.embedding_dimensions,
|
||||
}
|
||||
previous = self.store.active_manifest()
|
||||
|
||||
def document_sources(manifest: CorpusManifest) -> dict[str, dict]:
|
||||
return {
|
||||
document.document_id: {
|
||||
"document_id": document.document_id,
|
||||
"source_id": document.source_id,
|
||||
"source_uri": document.source_uri,
|
||||
"source_fingerprint": document.source_fingerprint,
|
||||
"modified_at": (
|
||||
document.modified_at.isoformat().replace("+00:00", "Z")
|
||||
if document.modified_at else None
|
||||
),
|
||||
"source_metadata": _canonical_json(document.metadata.get("source")),
|
||||
"media_type": document.media_type,
|
||||
"content_hash": document.content_hash,
|
||||
"pipeline_version": document.pipeline_version,
|
||||
}
|
||||
for document in manifest.documents
|
||||
}
|
||||
|
||||
def active_assets_are_valid(manifest: CorpusManifest | None) -> bool:
|
||||
if manifest is None or manifest.metadata.get("workspace_id") != workspace_id:
|
||||
return False
|
||||
actual_documents = {document.source_id: document for document in manifest.documents}
|
||||
persisted_snapshot = _canonical_json(manifest.metadata.get("source_snapshot"))
|
||||
if (
|
||||
not isinstance(persisted_snapshot, dict)
|
||||
or set(persisted_snapshot) != set(actual_documents)
|
||||
or manifest.metadata.get("compatibility_fingerprint") != compatibility
|
||||
or _canonical_json(manifest.metadata.get("document_sources"))
|
||||
!= document_sources(manifest)
|
||||
):
|
||||
return False
|
||||
for source_id, document in actual_documents.items():
|
||||
source_payload = persisted_snapshot[source_id]
|
||||
source = SourceObject.model_validate({
|
||||
"source_id": source_payload["source_id"],
|
||||
"uri": source_payload["uri"],
|
||||
"fingerprint": source_payload["fingerprint"],
|
||||
"modified_at": source_payload["modified_at"],
|
||||
"metadata": source_payload["metadata"],
|
||||
})
|
||||
content = self.store.read_document(document.document_id, manifest.manifest_id)
|
||||
expected_uri = canonical_provenance_uri(source.uri)
|
||||
expected_id = "doc:" + hashlib.sha256(
|
||||
f"{source.source_id}\n{expected_uri}".encode()
|
||||
).hexdigest()
|
||||
expected_media_type = source_payload.get("media_type")
|
||||
if (
|
||||
content != document.content
|
||||
or document.document_id != expected_id
|
||||
or document.source_id != source.source_id
|
||||
or document.source_uri != expected_uri
|
||||
or document.source_fingerprint != source.fingerprint
|
||||
or document.modified_at != source.modified_at
|
||||
or _canonical_json(document.metadata.get("source"))
|
||||
!= _canonical_json(source.metadata)
|
||||
or (
|
||||
isinstance(expected_media_type, str)
|
||||
and document.media_type != expected_media_type
|
||||
)
|
||||
or document.pipeline_version != self.pipeline_version
|
||||
):
|
||||
return False
|
||||
expected_chunks = tuple(
|
||||
part for document in manifest.documents for part in chunk(document, self.chunk_policy)
|
||||
)
|
||||
if any(document.content and not chunk(document, self.chunk_policy)
|
||||
for document in manifest.documents):
|
||||
return False
|
||||
if _canonical_json([part.model_dump(mode="json") for part in manifest.chunks]) != (
|
||||
_canonical_json([part.model_dump(mode="json") for part in expected_chunks])
|
||||
):
|
||||
return False
|
||||
generations = manifest.metadata.get("document_generations")
|
||||
if not isinstance(generations, Mapping):
|
||||
return False
|
||||
health = self.vector_store.health()
|
||||
if (
|
||||
not health.ok
|
||||
or health.dimension_compatible is not True
|
||||
or health.expected_dimension != self.embedding_dimensions
|
||||
or health.observed_dimensions != (self.embedding_dimensions,)
|
||||
):
|
||||
return False
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
for part in expected_chunks:
|
||||
generation = generations.get(part.document_id)
|
||||
if not isinstance(generation, str):
|
||||
return False
|
||||
record_id = f"{workspace_id}:{generation}:{part.chunk_id}"
|
||||
if existing.get(record_id) != part.content_hash:
|
||||
return False
|
||||
return True
|
||||
|
||||
try:
|
||||
active_assets_valid = active_assets_are_valid(previous)
|
||||
except Exception:
|
||||
active_assets_valid = False
|
||||
reusable = (
|
||||
active_assets_valid
|
||||
and _canonical_json(previous.metadata.get("source_snapshot")) == source_snapshot
|
||||
and _canonical_json(previous.metadata.get("job_binding")) == job_binding
|
||||
)
|
||||
if not dry_run and resume_run_id is None and reusable:
|
||||
return PipelineResult(
|
||||
"succeeded", previous.manifest_id, False, (),
|
||||
tuple(sorted(item.source_id for _, item in discovered)), (), previous,
|
||||
)
|
||||
spec = JobSpec(
|
||||
workspace_id=workspace_id,
|
||||
job_type="evidence",
|
||||
workspace_root=workspace_root,
|
||||
spec_version="jobs-v1",
|
||||
pipeline_version=self.pipeline_version,
|
||||
config_fingerprint=config_fingerprint,
|
||||
input_fingerprint=_fingerprint([input_fingerprint, discovered_fingerprint]),
|
||||
stage_ids=EVIDENCE_STAGE_IDS,
|
||||
dry_run=dry_run,
|
||||
resume_run_id=resume_run_id,
|
||||
)
|
||||
|
||||
def artifact(context: JobContext, name: str) -> Path:
|
||||
root = context.run_dir / "artifacts"
|
||||
root.mkdir(exist_ok=True)
|
||||
return root / name
|
||||
|
||||
def write(context: JobContext, name: str, value) -> None:
|
||||
artifact(context, name).write_text(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")), encoding="utf-8"
|
||||
)
|
||||
|
||||
def read(context: JobContext, name: str):
|
||||
try:
|
||||
return json.loads(artifact(context, name).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as error:
|
||||
raise PipelineError("preprocessing checkpoint artifact is corrupt") from error
|
||||
|
||||
def discover_stage(context: JobContext) -> None:
|
||||
previous = self.store.active_manifest()
|
||||
prior = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
||||
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
|
||||
previous_snapshot = (
|
||||
_canonical_json(previous.metadata.get("source_snapshot")) if previous else {}
|
||||
)
|
||||
rebuild = bool(previous and not active_assets_valid)
|
||||
changed = sorted(
|
||||
item.source_id for _, item in discovered
|
||||
if rebuild or item.source_id not in prior
|
||||
or previous_snapshot.get(item.source_id) != source_snapshot[item.source_id]
|
||||
)
|
||||
unchanged = sorted(set(fingerprints) - set(changed))
|
||||
removed = sorted(set(prior) - set(fingerprints))
|
||||
write(context, "plan.json", {
|
||||
"generation": f"gen:{context.run_id}",
|
||||
"compatibility": compatibility,
|
||||
"job_binding": job_binding,
|
||||
"source_snapshot": source_snapshot,
|
||||
"fingerprints": fingerprints,
|
||||
"changed": changed,
|
||||
"unchanged": unchanged,
|
||||
"removed": removed,
|
||||
"previous": previous.model_dump(mode="json") if previous else None,
|
||||
})
|
||||
return StageArtifacts(("plan.json",))
|
||||
|
||||
def acquire_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
plan = read(context, "plan.json")
|
||||
previous = CorpusManifest.model_validate(plan["previous"]) if plan["previous"] else None
|
||||
prior = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
||||
documents = [prior[source_id] for source_id in plan["unchanged"]]
|
||||
for source_id in plan["changed"]:
|
||||
source, item = source_by_id[source_id]
|
||||
documents.append(normalize(
|
||||
evidence_acquisition.acquire(source, item), self.pipeline_version,
|
||||
))
|
||||
documents.sort(key=lambda value: value.source_id)
|
||||
chunks = [part for document in documents for part in chunk(document, self.chunk_policy)]
|
||||
previous_generations = dict(previous.metadata.get("document_generations", {})) if previous else {}
|
||||
changed = set(plan["changed"])
|
||||
generations = {
|
||||
document.document_id: (
|
||||
plan["generation"] if document.source_id in changed
|
||||
else previous_generations.get(document.document_id, previous.vector_generation)
|
||||
) for document in documents
|
||||
}
|
||||
manifest = CorpusManifest(
|
||||
pipeline_version=self.pipeline_version,
|
||||
embedding_model=self.embedding_model,
|
||||
embedding_dimensions=self.embedding_dimensions,
|
||||
vector_generation=plan["generation"],
|
||||
documents=tuple(documents), chunks=tuple(chunks),
|
||||
metadata={
|
||||
"workspace_id": self.workspace_id,
|
||||
"compatibility_fingerprint": compatibility,
|
||||
"job_binding": plan["job_binding"],
|
||||
"source_snapshot": plan["source_snapshot"],
|
||||
"document_sources": {
|
||||
document.document_id: {
|
||||
"document_id": document.document_id,
|
||||
"source_id": document.source_id,
|
||||
"source_uri": document.source_uri,
|
||||
"source_fingerprint": document.source_fingerprint,
|
||||
"modified_at": (
|
||||
document.modified_at.isoformat().replace("+00:00", "Z")
|
||||
if document.modified_at else None
|
||||
),
|
||||
"source_metadata": _canonical_json(
|
||||
document.metadata.get("source")
|
||||
),
|
||||
"media_type": document.media_type,
|
||||
"content_hash": document.content_hash,
|
||||
"pipeline_version": document.pipeline_version,
|
||||
}
|
||||
for document in documents
|
||||
},
|
||||
"fingerprints": plan["fingerprints"],
|
||||
"removed": plan["removed"],
|
||||
"document_generations": generations,
|
||||
},
|
||||
)
|
||||
write(context, "manifest.json", manifest.model_dump(mode="json"))
|
||||
return StageArtifacts(("manifest.json",))
|
||||
|
||||
def embed_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
|
||||
parts = [part for part in manifest.chunks if part.document_id in changed_docs]
|
||||
embeddings = self.embedder.embed_documents([part.content for part in parts])
|
||||
if len(embeddings) != len(parts) or any(
|
||||
len(vector) != self.embedding_dimensions for vector in embeddings
|
||||
):
|
||||
raise PipelineError("embedding output is incompatible")
|
||||
write(context, "embeddings.json", embeddings)
|
||||
return StageArtifacts(("embeddings.json",))
|
||||
|
||||
def records(context: JobContext):
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
|
||||
parts = [part for part in manifest.chunks if part.document_id in changed_docs]
|
||||
embeddings = read(context, "embeddings.json")
|
||||
return [self._vector_record(part, vector, plan["generation"], self.workspace_id)
|
||||
for part, vector in zip(parts, embeddings, strict=True)]
|
||||
|
||||
def compensate(context: JobContext) -> None:
|
||||
generation = read(context, "plan.json")["generation"]
|
||||
if self.store.active_generation() != generation:
|
||||
self.store.discard(generation)
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
write(context, "compensated.json", {"generation": generation})
|
||||
|
||||
def rotate_compensated_generation(context: JobContext) -> None:
|
||||
marker = artifact(context, "compensated.json")
|
||||
if not marker.exists():
|
||||
return
|
||||
plan = read(context, "plan.json")
|
||||
old = plan["generation"]
|
||||
plan["generation"] = f"gen:{uuid.uuid4().hex}"
|
||||
write(context, "plan.json", plan)
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
changed = set(plan["changed"])
|
||||
generations = dict(manifest.metadata["document_generations"])
|
||||
for document in manifest.documents:
|
||||
if document.source_id in changed and generations.get(document.document_id) == old:
|
||||
generations[document.document_id] = plan["generation"]
|
||||
manifest_payload = manifest.model_dump(mode="json")
|
||||
manifest_payload["metadata"]["document_generations"] = generations
|
||||
manifest_payload["vector_generation"] = plan["generation"]
|
||||
manifest = CorpusManifest.model_validate(manifest_payload)
|
||||
write(context, "manifest.json", manifest.model_dump(mode="json"))
|
||||
marker.unlink()
|
||||
|
||||
def vector_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
rotate_compensated_generation(context)
|
||||
values = records(context)
|
||||
write(context, "vector-intent.json", {
|
||||
"generation": read(context, "plan.json")["generation"],
|
||||
"records": {value.record.id: value.content_hash for value in values},
|
||||
})
|
||||
seal_stage_artifacts(
|
||||
context, "vector_upsert",
|
||||
("plan.json", "manifest.json", "vector-intent.json"), spec,
|
||||
)
|
||||
try:
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
missing = [
|
||||
value for value in values
|
||||
if existing.get(value.record.id) != value.content_hash
|
||||
]
|
||||
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
|
||||
|
||||
def stage_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
recovered = False
|
||||
try:
|
||||
if artifact(context, "compensated.json").exists():
|
||||
recovered = True
|
||||
rotate_compensated_generation(context)
|
||||
values = records(context)
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
|
||||
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
plan = read(context, "plan.json")
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
if not self.store.generation_path(plan["generation"]).exists():
|
||||
self.store.stage(
|
||||
manifest, {doc.document_id: doc.content for doc in manifest.documents},
|
||||
generation=plan["generation"],
|
||||
)
|
||||
self.store.manifest(plan["generation"])
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
return StageArtifacts(
|
||||
("plan.json", "manifest.json", "vector-intent.json") if recovered else ()
|
||||
)
|
||||
|
||||
def publish_stage(context: JobContext) -> None:
|
||||
if context.dry_run:
|
||||
return StageArtifacts()
|
||||
if artifact(context, "compensated.json").exists():
|
||||
rotate_compensated_generation(context)
|
||||
values = records(context)
|
||||
try:
|
||||
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
|
||||
missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
|
||||
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
|
||||
generation = read(context, "plan.json")["generation"]
|
||||
try:
|
||||
if not self.store.generation_path(generation).exists():
|
||||
self.store.stage(
|
||||
manifest, {doc.document_id: doc.content for doc in manifest.documents},
|
||||
generation=generation,
|
||||
)
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
generation = read(context, "plan.json")["generation"]
|
||||
try:
|
||||
self.store.publish(generation)
|
||||
except Exception:
|
||||
compensate(context)
|
||||
raise
|
||||
return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
|
||||
|
||||
def retention_stage(context: JobContext) -> None:
|
||||
if not context.dry_run:
|
||||
self.gc(workspace_root=workspace_root)
|
||||
|
||||
report = run_job(spec, [
|
||||
discover_stage, acquire_stage, embed_stage, vector_stage,
|
||||
stage_stage, publish_stage, retention_stage,
|
||||
], after_stage_return=after_stage_return)
|
||||
run_dir = workspace_root / ".tht-jobs" / "evidence" / "runs" / report.run_id
|
||||
plan = json.loads((run_dir / "artifacts" / "plan.json").read_text())
|
||||
if dry_run:
|
||||
manifest = self.store.active_manifest() or CorpusManifest(pipeline_version=self.pipeline_version)
|
||||
generation = None
|
||||
published = False
|
||||
elif report.status == "succeeded":
|
||||
generation = plan["generation"]
|
||||
manifest = self.store.manifest(generation)
|
||||
published = True
|
||||
else:
|
||||
generation = plan["generation"]
|
||||
manifest_path = run_dir / "artifacts" / "manifest.json"
|
||||
manifest = (CorpusManifest.model_validate_json(manifest_path.read_text())
|
||||
if manifest_path.exists() else CorpusManifest(pipeline_version=self.pipeline_version))
|
||||
published = False
|
||||
return PipelineResult(
|
||||
report.status, generation, published, tuple(plan["changed"]),
|
||||
tuple(plan["unchanged"]), tuple(plan["removed"]), manifest,
|
||||
report.run_id, report.resumed_from,
|
||||
)
|
||||
|
||||
def _run(self, *, dry_run: bool = False, resume: str | None = None) -> PipelineResult:
|
||||
generation = None
|
||||
vector_written = False
|
||||
previous = self.store.active_manifest()
|
||||
try:
|
||||
discovered = self._discover()
|
||||
except Exception as error:
|
||||
raise PipelineError("Evidence discovery failed") from error
|
||||
prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {}
|
||||
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
|
||||
compatibility = _fingerprint({
|
||||
"pipeline": self.pipeline_version, "model": self.embedding_model,
|
||||
"dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy),
|
||||
})
|
||||
previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None
|
||||
rebuild = previous is not None and compatibility != previous_compatibility
|
||||
changed = tuple(item.source_id for _, item in discovered if rebuild or prior_documents.get(item.source_id) is None or prior_documents[item.source_id].source_fingerprint != item.fingerprint)
|
||||
unchanged = tuple(item.source_id for _, item in discovered if item.source_id not in changed)
|
||||
removed = tuple(sorted(set(prior_documents) - set(fingerprints)))
|
||||
if dry_run:
|
||||
manifest = previous or CorpusManifest(pipeline_version=self.pipeline_version)
|
||||
return PipelineResult("succeeded", None, False, changed, unchanged, removed, manifest)
|
||||
if previous is not None and not changed and not removed:
|
||||
return PipelineResult(
|
||||
"succeeded", previous.manifest_id, False, changed, unchanged, removed, previous
|
||||
)
|
||||
|
||||
documents: list[CanonicalDocument] = [prior_documents[source_id] for source_id in unchanged]
|
||||
changed_set = set(changed)
|
||||
try:
|
||||
for source, item in discovered:
|
||||
if item.source_id in changed_set:
|
||||
documents.append(normalize(
|
||||
evidence_acquisition.acquire(source, item), self.pipeline_version,
|
||||
))
|
||||
documents.sort(key=lambda document: document.source_id)
|
||||
chunks: list[CanonicalChunk] = []
|
||||
for document in documents:
|
||||
chunks.extend(chunk(document, self.chunk_policy))
|
||||
generation = resume or f"gen:{uuid.uuid4().hex}"
|
||||
previous_generations = dict(previous.metadata.get("document_generations", {})) if previous else {}
|
||||
document_generations = {
|
||||
document.document_id: (
|
||||
generation if document.source_id in changed_set
|
||||
else previous_generations.get(document.document_id, previous.vector_generation)
|
||||
)
|
||||
for document in documents
|
||||
}
|
||||
manifest = CorpusManifest(
|
||||
pipeline_version=self.pipeline_version,
|
||||
embedding_model=self.embedding_model,
|
||||
embedding_dimensions=self.embedding_dimensions,
|
||||
vector_generation=generation,
|
||||
documents=tuple(documents), chunks=tuple(chunks),
|
||||
metadata={
|
||||
"workspace_id": self.workspace_id,
|
||||
"compatibility_fingerprint": compatibility,
|
||||
"fingerprints": fingerprints,
|
||||
"removed": list(removed),
|
||||
"document_generations": document_generations,
|
||||
},
|
||||
)
|
||||
changed_documents = {document.document_id for document in documents if document.source_id in changed_set}
|
||||
changed_chunks = [part for part in chunks if part.document_id in changed_documents]
|
||||
embeddings = self.embedder.embed_documents([part.content for part in changed_chunks])
|
||||
if len(embeddings) != len(changed_chunks):
|
||||
raise PipelineError("embedding count mismatch")
|
||||
if any(len(vector) != self.embedding_dimensions for vector in embeddings):
|
||||
raise PipelineError("embedding dimension mismatch")
|
||||
records = [self._vector_record(part, vector, generation, self.workspace_id) for part, vector in zip(changed_chunks, embeddings, strict=True)]
|
||||
if records:
|
||||
written = self.vector_store.upsert("evidence", records)
|
||||
vector_written = True
|
||||
if written != len(records):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
generation_path = self.store.generation_path(generation)
|
||||
if resume is not None and generation_path.exists():
|
||||
staged_manifest = self.store.manifest(generation)
|
||||
expected = manifest.model_dump(mode="json", exclude={"created_at", "manifest_id"})
|
||||
actual = staged_manifest.model_dump(mode="json", exclude={"created_at", "manifest_id"})
|
||||
actual["metadata"].pop("files", None)
|
||||
if actual != expected:
|
||||
raise PipelineError("resume generation is incompatible")
|
||||
staged = generation
|
||||
else:
|
||||
staged = self.store.stage(
|
||||
manifest, {document.document_id: document.content for document in documents},
|
||||
generation=generation,
|
||||
)
|
||||
self.store.publish(staged)
|
||||
self.gc(workspace_root=self.store.root.parent)
|
||||
except PipelineError:
|
||||
self._compensate(generation, vector_written)
|
||||
raise
|
||||
except Exception as error:
|
||||
self._compensate(generation, vector_written)
|
||||
raise PipelineError("Evidence preprocessing failed") from error
|
||||
return PipelineResult("succeeded", generation, True, changed, unchanged, removed, self.store.manifest(generation))
|
||||
|
||||
def _compensate(self, generation: str | None, vector_written: bool) -> None:
|
||||
if generation is None:
|
||||
return
|
||||
try:
|
||||
self.store.discard(generation)
|
||||
except Exception:
|
||||
pass
|
||||
if vector_written:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _vector_record(
|
||||
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
|
||||
):
|
||||
record = VectorRecord(
|
||||
id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
|
||||
kind="evidence", ref=chunk.document_id,
|
||||
title=str(chunk.metadata.get("title", "")), content=chunk.content,
|
||||
metadata={
|
||||
**dict(chunk.metadata), "document_id": chunk.document_id,
|
||||
"workspace_id": workspace_id,
|
||||
"source_uri": chunk.source_uri, "ordinal": chunk.ordinal,
|
||||
"vector_generation": generation,
|
||||
},
|
||||
)
|
||||
return VectorWriteRecord(record=record, embedding=embedding, content_hash=chunk.content_hash)
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Evidence-owned immutable corpus generations and an atomic ACTIVE pointer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import fcntl
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import shutil
|
||||
import uuid
|
||||
import hashlib
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from contextlib import contextmanager
|
||||
|
||||
from tht.evidence.corpus.models import CorpusManifest
|
||||
|
||||
|
||||
_GENERATION = re.compile(r"^gen:[0-9a-f]{32}$")
|
||||
|
||||
|
||||
class UnsafeCorpusPath(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _atomic_write(path: Path, payload: bytes) -> None:
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
class CorpusStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
self.active_path = self.root / "ACTIVE"
|
||||
self._replace = os.replace
|
||||
self._fsync_directory = self._sync_root
|
||||
self._lock_state = threading.local()
|
||||
self._ensure_root()
|
||||
|
||||
def _ensure_root(self) -> None:
|
||||
if self.root.is_symlink():
|
||||
raise UnsafeCorpusPath("corpus root must not be a symlink")
|
||||
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
info = self.root.lstat()
|
||||
if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid():
|
||||
raise UnsafeCorpusPath("corpus root is unsafe")
|
||||
|
||||
@contextmanager
|
||||
def writer_lock(self):
|
||||
depth = getattr(self._lock_state, "depth", 0)
|
||||
if depth:
|
||||
self._lock_state.depth = depth + 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._lock_state.depth -= 1
|
||||
return
|
||||
lock_path = self.root / ".writer.lock"
|
||||
fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
|
||||
raise UnsafeCorpusPath("corpus writer lock is unsafe")
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
self._lock_state.depth = 1
|
||||
yield
|
||||
finally:
|
||||
self._lock_state.depth = 0
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
os.close(fd)
|
||||
|
||||
def generation_path(self, generation: str) -> Path:
|
||||
if not _GENERATION.fullmatch(generation):
|
||||
raise UnsafeCorpusPath("invalid corpus generation")
|
||||
path = self.root / generation.replace(":", "-")
|
||||
if path.is_symlink():
|
||||
raise UnsafeCorpusPath("generation must not be a symlink")
|
||||
return path
|
||||
|
||||
def stage(
|
||||
self,
|
||||
manifest: CorpusManifest,
|
||||
materialized: dict[str, str],
|
||||
*,
|
||||
generation: str | None = None,
|
||||
) -> str:
|
||||
generation = generation or f"gen:{uuid.uuid4().hex}"
|
||||
path = self.generation_path(generation)
|
||||
try:
|
||||
path.mkdir(mode=0o700)
|
||||
except FileExistsError:
|
||||
raise UnsafeCorpusPath("generation already exists") from None
|
||||
documents = path / "documents"
|
||||
documents.mkdir(mode=0o700)
|
||||
files: dict[str, str] = {}
|
||||
for document in manifest.documents:
|
||||
relative = f"documents/{document.document_id.removeprefix('doc:')}.md"
|
||||
_atomic_write(path / relative, materialized[document.document_id].encode("utf-8"))
|
||||
files[document.document_id] = relative
|
||||
payload = json.loads(manifest.model_dump_json())
|
||||
metadata = payload["metadata"]
|
||||
metadata["files"] = files
|
||||
payload.update({"manifest_id": generation, "metadata": metadata})
|
||||
staged = CorpusManifest.model_validate(payload)
|
||||
_atomic_write(path / "manifest.json", (staged.model_dump_json(indent=2) + "\n").encode())
|
||||
return generation
|
||||
|
||||
def publish(self, generation: str) -> str:
|
||||
manifest = self.manifest(generation)
|
||||
if manifest.manifest_id != generation:
|
||||
raise UnsafeCorpusPath("manifest generation mismatch")
|
||||
if self.active_generation() == generation:
|
||||
return generation
|
||||
previous = self.active_generation()
|
||||
published_marker = self.generation_path(generation) / "PUBLISHED"
|
||||
temporary = self.active_path.with_name(f".ACTIVE.{uuid.uuid4().hex}.tmp")
|
||||
replaced = False
|
||||
try:
|
||||
_atomic_write(temporary, (generation + "\n").encode())
|
||||
self._replace(temporary, self.active_path)
|
||||
replaced = True
|
||||
self._fsync_directory()
|
||||
_atomic_write(
|
||||
published_marker,
|
||||
(datetime.now(UTC).isoformat().replace("+00:00", "Z") + "\n").encode("ascii"),
|
||||
)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
if replaced:
|
||||
if previous is None:
|
||||
self.active_path.unlink(missing_ok=True)
|
||||
else:
|
||||
rollback = self.active_path.with_name(f".ACTIVE.rollback.{uuid.uuid4().hex}.tmp")
|
||||
_atomic_write(rollback, (previous + "\n").encode())
|
||||
self._replace(rollback, self.active_path)
|
||||
self._sync_root()
|
||||
raise
|
||||
return generation
|
||||
|
||||
def _sync_root(self) -> None:
|
||||
directory = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
|
||||
def active_generation(self) -> str | None:
|
||||
try:
|
||||
if self.active_path.is_symlink():
|
||||
raise UnsafeCorpusPath("ACTIVE must not be a symlink")
|
||||
value = self.active_path.read_text(encoding="ascii").strip()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if not _GENERATION.fullmatch(value):
|
||||
raise UnsafeCorpusPath("ACTIVE contains an invalid generation")
|
||||
return value
|
||||
|
||||
def manifest(self, generation: str) -> CorpusManifest:
|
||||
path = self.generation_path(generation)
|
||||
manifest_path = path / "manifest.json"
|
||||
if manifest_path.is_symlink():
|
||||
raise UnsafeCorpusPath("manifest must not be a symlink")
|
||||
return CorpusManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
|
||||
|
||||
def discard(self, generation: str) -> None:
|
||||
path = self.generation_path(generation)
|
||||
if path.exists():
|
||||
if path.is_symlink() or not stat.S_ISDIR(path.lstat().st_mode):
|
||||
raise UnsafeCorpusPath("generation cleanup target is unsafe")
|
||||
shutil.rmtree(path)
|
||||
|
||||
def active_manifest(self) -> CorpusManifest | None:
|
||||
generation = self.active_generation()
|
||||
return self.manifest(generation) if generation else None
|
||||
|
||||
def list_generations(self) -> list[str]:
|
||||
values = []
|
||||
for entry in self.root.iterdir():
|
||||
match = re.fullmatch(r"gen-([0-9a-f]{32})", entry.name)
|
||||
if match and not entry.is_symlink() and stat.S_ISDIR(entry.lstat().st_mode):
|
||||
values.append(f"gen:{match.group(1)}")
|
||||
return sorted(values, key=lambda value: self.generation_path(value).stat().st_mtime_ns)
|
||||
|
||||
def published_generations(self) -> list[str]:
|
||||
active = self.active_generation()
|
||||
published = []
|
||||
for generation in self.list_generations():
|
||||
path = self.generation_path(generation)
|
||||
marker = path / "PUBLISHED"
|
||||
if generation != active and not marker.is_file():
|
||||
continue
|
||||
try:
|
||||
manifest = self.manifest(generation)
|
||||
if manifest.manifest_id != generation:
|
||||
continue
|
||||
timestamp = marker.read_text(encoding="ascii").strip() if marker.is_file() else ""
|
||||
key = (timestamp or manifest.created_at.isoformat(), generation)
|
||||
published.append((key, generation))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return [generation for _, generation in sorted(published)]
|
||||
|
||||
def resolve_document(self, document_id: str, generation: str | None = None) -> Path | None:
|
||||
generation = generation or self.active_generation()
|
||||
if generation is None:
|
||||
return None
|
||||
manifest = self.manifest(generation)
|
||||
relative = manifest.metadata.get("files", {}).get(document_id)
|
||||
if not isinstance(relative, str):
|
||||
return None
|
||||
parts = Path(relative).parts
|
||||
if Path(relative).is_absolute() or parts[:1] != ("documents",) or len(parts) != 2:
|
||||
raise UnsafeCorpusPath("materialized document path is unsafe")
|
||||
return self.generation_path(generation) / relative
|
||||
|
||||
def read_document(self, document_id: str, generation: str | None = None) -> str | None:
|
||||
generation = generation or self.active_generation()
|
||||
if generation is None:
|
||||
return None
|
||||
manifest = self.manifest(generation)
|
||||
path = self.resolve_document(document_id, generation)
|
||||
document = next((item for item in manifest.documents if item.document_id == document_id), None)
|
||||
if path is None or document is None:
|
||||
return None
|
||||
generation_fd = os.open(self.generation_path(generation), os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
documents_fd = fd = None
|
||||
try:
|
||||
documents_fd = os.open("documents", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=generation_fd)
|
||||
fd = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=documents_fd)
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
|
||||
raise UnsafeCorpusPath("materialized document is unsafe")
|
||||
payload = os.read(fd, info.st_size + 1)
|
||||
if len(payload) != info.st_size or "sha256:" + hashlib.sha256(payload).hexdigest() != document.content_hash:
|
||||
raise UnsafeCorpusPath("materialized document hash mismatch")
|
||||
return payload.decode("utf-8")
|
||||
except (OSError, UnicodeError) as error:
|
||||
raise UnsafeCorpusPath("materialized document read failed") from error
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
if documents_fd is not None:
|
||||
os.close(documents_fd)
|
||||
os.close(generation_fd)
|
||||
|
||||
def materialize_document(
|
||||
self, document_id: str, destination: Path, generation: str | None = None,
|
||||
) -> Path | None:
|
||||
content = self.read_document(document_id, generation)
|
||||
if content is None:
|
||||
return None
|
||||
destination = Path(destination)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
_atomic_write(destination, content.encode("utf-8"))
|
||||
destination.chmod(0o400)
|
||||
return destination
|
||||
Reference in New Issue
Block a user