refactor(evidence): migrate acquisition and preprocessing (#30)

This commit is contained in:
2026-08-24 02:25:11 +02:00
parent d5e78febd3
commit 44f1efa5ba
18 changed files with 554 additions and 303 deletions
+9 -4
View File
@@ -15,7 +15,8 @@ from tht.corpus.chunk import ChunkPolicy, chunk
from tht.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
from tht.corpus.normalize import normalize
from tht.corpus.store import CorpusStore
from tht.ports.evidence import EvidenceSource, SourceObject, canonical_provenance_uri
import tht.evidence.acquisition as evidence_acquisition
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
from tht.ports.vector import VectorStore, VectorWriteRecord
from tht.vectorstore.records import VectorRecord
from tht.jobs.models import JobSpec
@@ -228,7 +229,7 @@ class CorpusPipeline:
discovered = []
seen = set()
for source in self.sources:
for item in source.discover():
for item in evidence_acquisition.discover(source):
if item.source_id in seen:
raise PipelineError("duplicate Evidence source identity")
seen.add(item.source_id)
@@ -455,7 +456,9 @@ class CorpusPipeline:
documents = [prior[source_id] for source_id in plan["unchanged"]]
for source_id in plan["changed"]:
source, item = source_by_id[source_id]
documents.append(normalize(source.acquire(item), self.pipeline_version))
documents.append(normalize(
evidence_acquisition.acquire(source, item), self.pipeline_version,
))
documents.sort(key=lambda value: value.source_id)
chunks = [part for document in documents for part in chunk(document, self.chunk_policy)]
previous_generations = dict(previous.metadata.get("document_generations", {})) if previous else {}
@@ -710,7 +713,9 @@ class CorpusPipeline:
try:
for source, item in discovered:
if item.source_id in changed_set:
documents.append(normalize(source.acquire(item), self.pipeline_version))
documents.append(normalize(
evidence_acquisition.acquire(source, item), self.pipeline_version,
))
documents.sort(key=lambda document: document.source_id)
chunks: list[CanonicalChunk] = []
for document in documents: