feat(preprocess): publish incremental Evidence corpus

2026-07-12 04:26:24 +02:00
parent 16a8bd9df6
commit b96d4f13b9
10 changed files with 708 additions and 0 deletions
@@ -0,0 +1,132 @@
+import pytest
+
+from tht.corpus.chunk import ChunkPolicy
+from tht.corpus.pipeline import CorpusPipeline, PipelineError
+from tht.corpus.store import CorpusStore
+from tht.ports.evidence import AcquiredDocument, SourceObject
+from tht.ports.vector import VectorCapabilities
+
+
+class Source:
+    def __init__(self, documents):
+        self.documents = documents
+        self.acquire_calls = []
+
+    def discover(self):
+        return [item[0] for item in self.documents]
+
+    def acquire(self, item):
+        self.acquire_calls.append(item.source_id)
+        payload = next(payload for source, payload in self.documents if source.source_id == item.source_id)
+        if isinstance(payload, Exception):
+            raise payload
+        return AcquiredDocument(source=item, content=payload.encode())
+
+
+class Embedder:
+    def __init__(self, dim=3, fail=False):
+        self.dim = dim
+        self.fail = fail
+        self.calls = []
+
+    def embed_documents(self, texts):
+        self.calls.extend(texts)
+        if self.fail:
+            raise RuntimeError("embed failed")
+        return [[float(i) for i in range(self.dim)] for _ in texts]
+
+
+class Vectors:
+    capabilities = VectorCapabilities(search=True, existing_hashes=True, upsert=True)
+
+    def __init__(self, fail=False):
+        self.fail = fail
+        self.records = []
+
+    def upsert(self, collection, records):
+        self.records.extend(records[:1] if self.fail else records)
+        if self.fail:
+            raise RuntimeError("partial write")
+        return len(records)
+
+
+def item(name, fingerprint):
+    return SourceObject(
+        source_id=f"fs:{name}", uri=f"file:///safe/{name}.md", fingerprint=f"sha256:{fingerprint}"
+    )
+
+
+def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None):
+    return CorpusPipeline(
+        store=CorpusStore(tmp_path / "corpus"), sources=[source],
+        embedder=embedder or Embedder(), vector_store=vectors or Vectors(),
+        embedding_model=model, embedding_dimensions=3,
+        chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100),
+        pipeline_version="evidence-v1",
+    )
+
+
+def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path):
+    one = item("one", "a")
+    first_source = Source([(one, "hello")])
+    first = pipeline(tmp_path, first_source)
+    first.run()
+    second_source = Source([(one, "ignored")])
+    second_embedder = Embedder()
+    result = pipeline(tmp_path, second_source, embedder=second_embedder).run()
+    assert result.unchanged == ("fs:one",)
+    assert second_source.acquire_calls == []
+    assert second_embedder.calls == []
+
+
+def test_removed_documents_are_marked_and_absent_from_new_manifest(tmp_path):
+    one, two = item("one", "a"), item("two", "b")
+    pipeline(tmp_path, Source([(one, "one"), (two, "two")])).run()
+    result = pipeline(tmp_path, Source([(one, "one")])).run()
+    assert result.removed == ("fs:two",)
+    assert {doc.source_id for doc in result.manifest.documents} == {"fs:one"}
+
+
+def test_model_or_chunk_policy_change_forces_full_rebuild(tmp_path):
+    one = item("one", "a")
+    pipeline(tmp_path, Source([(one, "hello")])).run()
+    source = Source([(one, "hello")])
+    changed = pipeline(tmp_path, source, model="model-b").run()
+    assert changed.changed == ("fs:one",)
+    assert source.acquire_calls == ["fs:one"]
+
+
+def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path):
+    one = item("one", "a")
+    good = pipeline(tmp_path, Source([(one, "old")]))
+    old = good.run().generation
+    changed = item("one", "b")
+    vectors = Vectors(fail=True)
+    broken = pipeline(tmp_path, Source([(changed, "new")]), vectors=vectors)
+    with pytest.raises(PipelineError):
+        broken.run()
+    assert broken.store.active_generation() == old
+    assert vectors.records[0].record.metadata["vector_generation"] != old
+
+
+def test_dimension_mismatch_fails_before_vector_write_and_publish(tmp_path):
+    one = item("one", "a")
+    vectors = Vectors()
+    candidate = pipeline(tmp_path, Source([(one, "hello")]), embedder=Embedder(dim=2), vectors=vectors)
+    with pytest.raises(PipelineError, match="dimension"):
+        candidate.run()
+    assert vectors.records == []
+    assert candidate.store.active_generation() is None
+
+
+def test_dry_run_and_failed_acquire_never_change_active(tmp_path):
+    one = item("one", "a")
+    active = pipeline(tmp_path, Source([(one, "old")])).run().generation
+    changed = item("one", "b")
+    dry = pipeline(tmp_path, Source([(changed, "new")])).run(dry_run=True)
+    assert dry.published is False
+    assert dry.generation is None
+    assert dry.manifest.documents[0].content == "old"
+    with pytest.raises(PipelineError):
+        pipeline(tmp_path, Source([(changed, RuntimeError("boom"))])).run()
+    assert CorpusStore(tmp_path / "corpus").active_generation() == active
@@ -0,0 +1,43 @@
+import pytest
+
+from tht.corpus.models import CorpusManifest
+from tht.corpus.store import CorpusStore, UnsafeCorpusPath
+
+
+def test_publish_switches_active_atomically_and_resolves_materialized_files(tmp_path):
+    store = CorpusStore(tmp_path / "corpus")
+    generation = store.stage(CorpusManifest(), {})
+    seen = []
+    store._replace = lambda source, target: (seen.append(source.read_text()), source.replace(target))
+    published = store.publish(generation)
+    assert published == generation
+    assert store.active_generation() == generation
+    assert seen == [generation + "\n"]
+
+
+def test_active_manifest_is_a_consistent_reader_snapshot(tmp_path):
+    store = CorpusStore(tmp_path / "corpus")
+    first = store.stage(CorpusManifest(metadata={"name": "first"}), {})
+    second = store.stage(CorpusManifest(metadata={"name": "second"}), {})
+    store.publish(first)
+    snapshot = store.active_manifest()
+    store.publish(second)
+    assert snapshot.metadata["name"] == "first"
+    assert store.active_manifest().metadata["name"] == "second"
+
+
+def test_store_rejects_symlinked_generation_root(tmp_path):
+    outside = tmp_path / "outside"
+    outside.mkdir()
+    root = tmp_path / "corpus"
+    root.symlink_to(outside, target_is_directory=True)
+    with pytest.raises(UnsafeCorpusPath):
+        CorpusStore(root)
+
+
+def test_active_pointer_cannot_escape_generation_root(tmp_path):
+    store = CorpusStore(tmp_path / "corpus")
+    store.root.mkdir(parents=True, exist_ok=True)
+    store.active_path.write_text("../outside\n")
+    with pytest.raises(UnsafeCorpusPath):
+        store.active_manifest()
@@ -0,0 +1,32 @@
+import json
+from types import SimpleNamespace
+
+from typer.testing import CliRunner
+
+from tht.cli import app
+
+
+def test_preprocess_evidence_json_is_pristine(monkeypatch, tmp_path):
+    import tht.cli.preprocess_cmd as command
+
+    result = SimpleNamespace(model_dump=lambda mode=None: {
+        "status": "succeeded", "generation": "gen:abc", "published": True
+    })
+    monkeypatch.setattr(command, "run_from_config", lambda *args, **kwargs: result)
+    response = CliRunner().invoke(
+        app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
+    )
+    assert response.exit_code == 0, response.output
+    assert json.loads(response.output)["generation"] == "gen:abc"
+
+
+def test_preprocess_failure_is_structured_and_nonzero(monkeypatch, tmp_path):
+    import tht.cli.preprocess_cmd as command
+
+    monkeypatch.setattr(command, "run_from_config", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("secret detail")))
+    response = CliRunner().invoke(
+        app, ["preprocess", "evidence", "--json", "-c", str(tmp_path / "workspace.yaml")]
+    )
+    assert response.exit_code != 0
+    assert json.loads(response.output) == {"status": "failed", "error": "preprocessing failed"}
+    assert "secret detail" not in response.output