import pytest from tht.corpus.chunk import ChunkPolicy from tht.corpus.pipeline import CorpusPipeline, PipelineError from tht.corpus.store import CorpusStore from tht.ports.evidence import AcquiredDocument, SourceObject from tht.ports.vector import VectorCapabilities class Source: def __init__(self, documents): self.documents = documents self.acquire_calls = [] def discover(self): return [item[0] for item in self.documents] def acquire(self, item): self.acquire_calls.append(item.source_id) payload = next(payload for source, payload in self.documents if source.source_id == item.source_id) if isinstance(payload, Exception): raise payload return AcquiredDocument(source=item, content=payload.encode()) class Embedder: def __init__(self, dim=3, fail=False): self.dim = dim self.fail = fail self.calls = [] def embed_documents(self, texts): self.calls.extend(texts) if self.fail: raise RuntimeError("embed failed") return [[float(i) for i in range(self.dim)] for _ in texts] class Vectors: capabilities = VectorCapabilities(search=True, existing_hashes=True, upsert=True) def __init__(self, fail=False): self.fail = fail self.records = [] def upsert(self, collection, records): self.records.extend(records[:1] if self.fail else records) if self.fail: raise RuntimeError("partial write") return len(records) def item(name, fingerprint): return SourceObject( source_id=f"fs:{name}", uri=f"file:///safe/{name}.md", fingerprint=f"sha256:{fingerprint}" ) def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None): return CorpusPipeline( store=CorpusStore(tmp_path / "corpus"), sources=[source], embedder=embedder or Embedder(), vector_store=vectors or Vectors(), embedding_model=model, embedding_dimensions=3, chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100), pipeline_version="evidence-v1", ) def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path): one = item("one", "a") first_source = Source([(one, "hello")]) first = pipeline(tmp_path, first_source) first.run() second_source = Source([(one, "ignored")]) second_embedder = Embedder() result = pipeline(tmp_path, second_source, embedder=second_embedder).run() assert result.unchanged == ("fs:one",) assert second_source.acquire_calls == [] assert second_embedder.calls == [] def test_removed_documents_are_marked_and_absent_from_new_manifest(tmp_path): one, two = item("one", "a"), item("two", "b") pipeline(tmp_path, Source([(one, "one"), (two, "two")])).run() result = pipeline(tmp_path, Source([(one, "one")])).run() assert result.removed == ("fs:two",) assert {doc.source_id for doc in result.manifest.documents} == {"fs:one"} def test_model_or_chunk_policy_change_forces_full_rebuild(tmp_path): one = item("one", "a") pipeline(tmp_path, Source([(one, "hello")])).run() source = Source([(one, "hello")]) changed = pipeline(tmp_path, source, model="model-b").run() assert changed.changed == ("fs:one",) assert source.acquire_calls == ["fs:one"] def test_partial_vector_failure_never_changes_active_or_exposes_generation(tmp_path): one = item("one", "a") good = pipeline(tmp_path, Source([(one, "old")])) old = good.run().generation changed = item("one", "b") vectors = Vectors(fail=True) broken = pipeline(tmp_path, Source([(changed, "new")]), vectors=vectors) with pytest.raises(PipelineError): broken.run() assert broken.store.active_generation() == old assert vectors.records[0].record.metadata["vector_generation"] != old def test_dimension_mismatch_fails_before_vector_write_and_publish(tmp_path): one = item("one", "a") vectors = Vectors() candidate = pipeline(tmp_path, Source([(one, "hello")]), embedder=Embedder(dim=2), vectors=vectors) with pytest.raises(PipelineError, match="dimension"): candidate.run() assert vectors.records == [] assert candidate.store.active_generation() is None def test_dry_run_and_failed_acquire_never_change_active(tmp_path): one = item("one", "a") active = pipeline(tmp_path, Source([(one, "old")])).run().generation changed = item("one", "b") dry = pipeline(tmp_path, Source([(changed, "new")])).run(dry_run=True) assert dry.published is False assert dry.generation is None assert dry.manifest.documents[0].content == "old" with pytest.raises(PipelineError): pipeline(tmp_path, Source([(changed, RuntimeError("boom"))])).run() assert CorpusStore(tmp_path / "corpus").active_generation() == active