refactor(evidence): remove legacy Python layout (#31)

2026-08-24 02:36:30 +02:00
parent 44f1efa5ba
commit 840848df94
36 changed files with 246 additions and 321 deletions
@@ -0,0 +1,272 @@
+"""Evidence-owned immutable corpus generations and an atomic ACTIVE pointer."""
+
+from __future__ import annotations
+
+import json
+import fcntl
+import os
+import re
+import stat
+import shutil
+import uuid
+import hashlib
+import threading
+from datetime import UTC, datetime
+from pathlib import Path
+from contextlib import contextmanager
+
+from tht.evidence.corpus.models import CorpusManifest
+
+
+_GENERATION = re.compile(r"^gen:[0-9a-f]{32}$")
+
+
+class UnsafeCorpusPath(RuntimeError):
+    pass
+
+
+def _atomic_write(path: Path, payload: bytes) -> None:
+    temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+    fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    try:
+        with os.fdopen(fd, "wb") as stream:
+            stream.write(payload)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+        directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+        try:
+            os.fsync(directory)
+        finally:
+            os.close(directory)
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+class CorpusStore:
+    def __init__(self, root: Path) -> None:
+        self.root = Path(root)
+        self.active_path = self.root / "ACTIVE"
+        self._replace = os.replace
+        self._fsync_directory = self._sync_root
+        self._lock_state = threading.local()
+        self._ensure_root()
+
+    def _ensure_root(self) -> None:
+        if self.root.is_symlink():
+            raise UnsafeCorpusPath("corpus root must not be a symlink")
+        self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
+        info = self.root.lstat()
+        if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid():
+            raise UnsafeCorpusPath("corpus root is unsafe")
+
+    @contextmanager
+    def writer_lock(self):
+        depth = getattr(self._lock_state, "depth", 0)
+        if depth:
+            self._lock_state.depth = depth + 1
+            try:
+                yield
+            finally:
+                self._lock_state.depth -= 1
+            return
+        lock_path = self.root / ".writer.lock"
+        fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
+        try:
+            info = os.fstat(fd)
+            if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
+                raise UnsafeCorpusPath("corpus writer lock is unsafe")
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            self._lock_state.depth = 1
+            yield
+        finally:
+            self._lock_state.depth = 0
+            fcntl.flock(fd, fcntl.LOCK_UN)
+            os.close(fd)
+
+    def generation_path(self, generation: str) -> Path:
+        if not _GENERATION.fullmatch(generation):
+            raise UnsafeCorpusPath("invalid corpus generation")
+        path = self.root / generation.replace(":", "-")
+        if path.is_symlink():
+            raise UnsafeCorpusPath("generation must not be a symlink")
+        return path
+
+    def stage(
+        self,
+        manifest: CorpusManifest,
+        materialized: dict[str, str],
+        *,
+        generation: str | None = None,
+    ) -> str:
+        generation = generation or f"gen:{uuid.uuid4().hex}"
+        path = self.generation_path(generation)
+        try:
+            path.mkdir(mode=0o700)
+        except FileExistsError:
+            raise UnsafeCorpusPath("generation already exists") from None
+        documents = path / "documents"
+        documents.mkdir(mode=0o700)
+        files: dict[str, str] = {}
+        for document in manifest.documents:
+            relative = f"documents/{document.document_id.removeprefix('doc:')}.md"
+            _atomic_write(path / relative, materialized[document.document_id].encode("utf-8"))
+            files[document.document_id] = relative
+        payload = json.loads(manifest.model_dump_json())
+        metadata = payload["metadata"]
+        metadata["files"] = files
+        payload.update({"manifest_id": generation, "metadata": metadata})
+        staged = CorpusManifest.model_validate(payload)
+        _atomic_write(path / "manifest.json", (staged.model_dump_json(indent=2) + "\n").encode())
+        return generation
+
+    def publish(self, generation: str) -> str:
+        manifest = self.manifest(generation)
+        if manifest.manifest_id != generation:
+            raise UnsafeCorpusPath("manifest generation mismatch")
+        if self.active_generation() == generation:
+            return generation
+        previous = self.active_generation()
+        published_marker = self.generation_path(generation) / "PUBLISHED"
+        temporary = self.active_path.with_name(f".ACTIVE.{uuid.uuid4().hex}.tmp")
+        replaced = False
+        try:
+            _atomic_write(temporary, (generation + "\n").encode())
+            self._replace(temporary, self.active_path)
+            replaced = True
+            self._fsync_directory()
+            _atomic_write(
+                published_marker,
+                (datetime.now(UTC).isoformat().replace("+00:00", "Z") + "\n").encode("ascii"),
+            )
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            if replaced:
+                if previous is None:
+                    self.active_path.unlink(missing_ok=True)
+                else:
+                    rollback = self.active_path.with_name(f".ACTIVE.rollback.{uuid.uuid4().hex}.tmp")
+                    _atomic_write(rollback, (previous + "\n").encode())
+                    self._replace(rollback, self.active_path)
+                self._sync_root()
+            raise
+        return generation
+
+    def _sync_root(self) -> None:
+        directory = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+        try:
+            os.fsync(directory)
+        finally:
+            os.close(directory)
+
+    def active_generation(self) -> str | None:
+        try:
+            if self.active_path.is_symlink():
+                raise UnsafeCorpusPath("ACTIVE must not be a symlink")
+            value = self.active_path.read_text(encoding="ascii").strip()
+        except FileNotFoundError:
+            return None
+        if not _GENERATION.fullmatch(value):
+            raise UnsafeCorpusPath("ACTIVE contains an invalid generation")
+        return value
+
+    def manifest(self, generation: str) -> CorpusManifest:
+        path = self.generation_path(generation)
+        manifest_path = path / "manifest.json"
+        if manifest_path.is_symlink():
+            raise UnsafeCorpusPath("manifest must not be a symlink")
+        return CorpusManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
+
+    def discard(self, generation: str) -> None:
+        path = self.generation_path(generation)
+        if path.exists():
+            if path.is_symlink() or not stat.S_ISDIR(path.lstat().st_mode):
+                raise UnsafeCorpusPath("generation cleanup target is unsafe")
+            shutil.rmtree(path)
+
+    def active_manifest(self) -> CorpusManifest | None:
+        generation = self.active_generation()
+        return self.manifest(generation) if generation else None
+
+    def list_generations(self) -> list[str]:
+        values = []
+        for entry in self.root.iterdir():
+            match = re.fullmatch(r"gen-([0-9a-f]{32})", entry.name)
+            if match and not entry.is_symlink() and stat.S_ISDIR(entry.lstat().st_mode):
+                values.append(f"gen:{match.group(1)}")
+        return sorted(values, key=lambda value: self.generation_path(value).stat().st_mtime_ns)
+
+    def published_generations(self) -> list[str]:
+        active = self.active_generation()
+        published = []
+        for generation in self.list_generations():
+            path = self.generation_path(generation)
+            marker = path / "PUBLISHED"
+            if generation != active and not marker.is_file():
+                continue
+            try:
+                manifest = self.manifest(generation)
+                if manifest.manifest_id != generation:
+                    continue
+                timestamp = marker.read_text(encoding="ascii").strip() if marker.is_file() else ""
+                key = (timestamp or manifest.created_at.isoformat(), generation)
+                published.append((key, generation))
+            except (OSError, ValueError):
+                continue
+        return [generation for _, generation in sorted(published)]
+
+    def resolve_document(self, document_id: str, generation: str | None = None) -> Path | None:
+        generation = generation or self.active_generation()
+        if generation is None:
+            return None
+        manifest = self.manifest(generation)
+        relative = manifest.metadata.get("files", {}).get(document_id)
+        if not isinstance(relative, str):
+            return None
+        parts = Path(relative).parts
+        if Path(relative).is_absolute() or parts[:1] != ("documents",) or len(parts) != 2:
+            raise UnsafeCorpusPath("materialized document path is unsafe")
+        return self.generation_path(generation) / relative
+
+    def read_document(self, document_id: str, generation: str | None = None) -> str | None:
+        generation = generation or self.active_generation()
+        if generation is None:
+            return None
+        manifest = self.manifest(generation)
+        path = self.resolve_document(document_id, generation)
+        document = next((item for item in manifest.documents if item.document_id == document_id), None)
+        if path is None or document is None:
+            return None
+        generation_fd = os.open(self.generation_path(generation), os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+        documents_fd = fd = None
+        try:
+            documents_fd = os.open("documents", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=generation_fd)
+            fd = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=documents_fd)
+            info = os.fstat(fd)
+            if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
+                raise UnsafeCorpusPath("materialized document is unsafe")
+            payload = os.read(fd, info.st_size + 1)
+            if len(payload) != info.st_size or "sha256:" + hashlib.sha256(payload).hexdigest() != document.content_hash:
+                raise UnsafeCorpusPath("materialized document hash mismatch")
+            return payload.decode("utf-8")
+        except (OSError, UnicodeError) as error:
+            raise UnsafeCorpusPath("materialized document read failed") from error
+        finally:
+            if fd is not None:
+                os.close(fd)
+            if documents_fd is not None:
+                os.close(documents_fd)
+            os.close(generation_fd)
+
+    def materialize_document(
+        self, document_id: str, destination: Path, generation: str | None = None,
+    ) -> Path | None:
+        content = self.read_document(document_id, generation)
+        if content is None:
+            return None
+        destination = Path(destination)
+        destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        _atomic_write(destination, content.encode("utf-8"))
+        destination.chmod(0o400)
+        return destination