feat(evidence): add filesystem and HTTP sources

harness/tests/test_config_resources.py

@@ -8,6 +8,8 @@ from tht.config import (
     ThothVectorHttpConfig,
     load_config,
 )
+from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
+from tht.adapters.factory import build_evidence_sources
 
 
 def test_direct_vector_passwords_load_from_file_references(monkeypatch, tmp_path):
@@ -130,3 +132,39 @@ embeddings: {base_url: http://ollama:11434, dim: 768}
     cfg = load_config(workspace)
     assert cfg.vectors.reader is None
     assert cfg.vectors.writer.api_key == "writer"
+
+
+def test_builds_typed_evidence_sources_and_keeps_legacy_compatible(tmp_path):
+    common = """
+dwh:
+  type: postgres_direct
+  connection: {database: d, schema: public, user: u, password: p}
+"""
+    modern = tmp_path / "modern.yaml"
+    modern.write_text(common + f"""
+evidence:
+  sources:
+    - type: filesystem
+      root: {tmp_path}
+      max_bytes: 123
+    - type: http
+      urls: ['https://example.test/doc.md?token=transport-only']
+""")
+    cfg = load_config(modern)
+    assert "transport-only" not in repr(cfg.evidence)
+    assert "transport-only" not in cfg.evidence.model_dump_json()
+    sources = build_evidence_sources(cfg)
+    assert isinstance(sources[0], FilesystemEvidenceSource)
+    assert isinstance(sources[1], HttpManifestEvidenceSource)
+    assert "transport-only" not in repr(sources[1])
+
+    legacy = tmp_path / "legacy.yaml"
+    (tmp_path / "curated").mkdir()
+    legacy.write_text(common + f"""
+evidence:
+  source_root: {tmp_path}
+  evidence_dir: curated
+""")
+    legacy_source = build_evidence_sources(load_config(legacy))[0]
+    assert isinstance(legacy_source, FilesystemEvidenceSource)
+    assert legacy_source.root == (tmp_path / "curated").resolve()

harness/tests/test_filesystem_evidence_source.py

@@ -0,0 +1,49 @@
+import pytest
+
+from tht.adapters.evidence import FilesystemEvidenceSource
+from tht.ports.evidence import EvidenceSourceError
+
+
+def test_filesystem_discovery_is_stable_and_acquisition_is_bounded(tmp_path):
+    (tmp_path / "z.md").write_text("z")
+    (tmp_path / "nested").mkdir()
+    (tmp_path / "nested" / "a.md").write_text("alpha")
+    source = FilesystemEvidenceSource(tmp_path, max_bytes=5)
+
+    first = list(source.discover())
+    assert [item.uri for item in first] == sorted(item.uri for item in first)
+    assert all(item.source_id.startswith("filesystem:") for item in first)
+    assert all(item.fingerprint.startswith("sha256:") for item in first)
+    assert source.acquire(first[0]).content in {b"alpha", b"z"}
+
+    (tmp_path / "large.md").write_bytes(b"123456")
+    with pytest.raises(EvidenceSourceError) as caught:
+        list(source.discover())
+    assert not caught.value.retryable
+    assert "large.md" not in str(caught.value)
+
+
+def test_filesystem_rejects_symlink_escape(tmp_path):
+    root = tmp_path / "root"
+    root.mkdir()
+    outside = tmp_path / "secret.md"
+    outside.write_text("secret")
+    (root / "escape.md").symlink_to(outside)
+
+    with pytest.raises(EvidenceSourceError) as caught:
+        list(FilesystemEvidenceSource(root).discover())
+    assert not caught.value.retryable
+    assert str(outside) not in str(caught.value)
+
+
+def test_filesystem_acquire_rejects_object_from_another_source(tmp_path):
+    left = tmp_path / "left"
+    right = tmp_path / "right"
+    left.mkdir()
+    right.mkdir()
+    (left / "doc.md").write_text("left")
+    (right / "doc.md").write_text("right")
+    item = next(iter(FilesystemEvidenceSource(left).discover()))
+
+    with pytest.raises(EvidenceSourceError):
+        FilesystemEvidenceSource(right).acquire(item)

harness/tests/test_http_evidence_source.py

@@ -0,0 +1,109 @@
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+from tht.adapters.evidence import HttpManifestEvidenceSource
+from tht.ports.evidence import EvidenceSourceError
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path.startswith("/etag"):
+            self.send_response(200)
+            self.send_header("ETag", '"abc"')
+            self.send_header("Content-Type", "text/markdown")
+            self.end_headers()
+            self.wfile.write(b"hello")
+        elif self.path == "/large":
+            self.send_response(200)
+            self.send_header("Content-Length", "20")
+            self.end_headers()
+            self.wfile.write(b"x" * 20)
+        elif self.path == "/busy":
+            self.send_response(503)
+            self.end_headers()
+        elif self.path == "/missing":
+            self.send_response(404)
+            self.end_headers()
+        elif self.path == "/redirect-private":
+            self.send_response(302)
+            self.send_header("Location", f"http://127.0.0.1:{self.server.server_port}/etag")
+            self.end_headers()
+        else:
+            self.send_response(200)
+            self.send_header("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT")
+            self.end_headers()
+            self.wfile.write(b"fallback")
+
+    def log_message(self, format, *args):
+        pass
+
+
+@pytest.fixture
+def server_url():
+    server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        yield f"http://127.0.0.1:{server.server_port}"
+    finally:
+        server.shutdown()
+        thread.join()
+
+
+def test_http_uses_etag_and_strips_query_from_provenance(server_url):
+    source = HttpManifestEvidenceSource([f"{server_url}/etag?token=secret"])
+    item = next(iter(source.discover()))
+
+    assert item.fingerprint.startswith("etag:")
+    assert item.fingerprint != "etag:abc"
+    assert item.uri == f"{server_url}/etag"
+    assert "secret" not in item.model_dump_json()
+    assert source.acquire(item).content == b"hello"
+
+
+def test_http_uses_last_modified_then_content_hash(server_url):
+    modified = next(iter(HttpManifestEvidenceSource([f"{server_url}/modified"]).discover()))
+    assert modified.fingerprint.startswith("last-modified:")
+
+    class NoValidators(Handler):
+        def do_GET(self):
+            self.send_response(200)
+            self.end_headers()
+            self.wfile.write(b"content")
+
+    server = ThreadingHTTPServer(("127.0.0.1", 0), NoValidators)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        item = next(iter(HttpManifestEvidenceSource(
+            [f"http://127.0.0.1:{server.server_port}/doc"]
+        ).discover()))
+        assert item.fingerprint.startswith("sha256:")
+    finally:
+        server.shutdown()
+        thread.join()
+
+
+@pytest.mark.parametrize("path,retryable", [("/busy", True), ("/missing", False)])
+def test_http_classifies_status_errors(server_url, path, retryable):
+    with pytest.raises(EvidenceSourceError) as caught:
+        list(HttpManifestEvidenceSource([server_url + path]).discover())
+    assert caught.value.retryable is retryable
+    assert server_url not in str(caught.value)
+
+
+def test_http_rejects_oversize_and_private_redirect(server_url):
+    with pytest.raises(EvidenceSourceError) as large:
+        list(HttpManifestEvidenceSource([server_url + "/large"], max_bytes=10).discover())
+    assert not large.value.retryable
+
+    with pytest.raises(EvidenceSourceError) as redirect:
+        list(HttpManifestEvidenceSource([server_url + "/redirect-private"]).discover())
+    assert not redirect.value.retryable
+
+
+def test_http_rejects_unsupported_manifest_scheme():
+    with pytest.raises(ValueError, match="http"):
+        HttpManifestEvidenceSource(["file:///tmp/secret"])