refactor(evidence): migrate acquisition and preprocessing (#30)
This commit is contained in:
@@ -15,7 +15,8 @@ from tht.corpus.chunk import ChunkPolicy, chunk
|
||||
from tht.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
|
||||
from tht.corpus.normalize import normalize
|
||||
from tht.corpus.store import CorpusStore
|
||||
from tht.ports.evidence import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
import tht.evidence.acquisition as evidence_acquisition
|
||||
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
from tht.ports.vector import VectorStore, VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
from tht.jobs.models import JobSpec
|
||||
@@ -228,7 +229,7 @@ class CorpusPipeline:
|
||||
discovered = []
|
||||
seen = set()
|
||||
for source in self.sources:
|
||||
for item in source.discover():
|
||||
for item in evidence_acquisition.discover(source):
|
||||
if item.source_id in seen:
|
||||
raise PipelineError("duplicate Evidence source identity")
|
||||
seen.add(item.source_id)
|
||||
@@ -455,7 +456,9 @@ class CorpusPipeline:
|
||||
documents = [prior[source_id] for source_id in plan["unchanged"]]
|
||||
for source_id in plan["changed"]:
|
||||
source, item = source_by_id[source_id]
|
||||
documents.append(normalize(source.acquire(item), self.pipeline_version))
|
||||
documents.append(normalize(
|
||||
evidence_acquisition.acquire(source, item), self.pipeline_version,
|
||||
))
|
||||
documents.sort(key=lambda value: value.source_id)
|
||||
chunks = [part for document in documents for part in chunk(document, self.chunk_policy)]
|
||||
previous_generations = dict(previous.metadata.get("document_generations", {})) if previous else {}
|
||||
@@ -710,7 +713,9 @@ class CorpusPipeline:
|
||||
try:
|
||||
for source, item in discovered:
|
||||
if item.source_id in changed_set:
|
||||
documents.append(normalize(source.acquire(item), self.pipeline_version))
|
||||
documents.append(normalize(
|
||||
evidence_acquisition.acquire(source, item), self.pipeline_version,
|
||||
))
|
||||
documents.sort(key=lambda document: document.source_id)
|
||||
chunks: list[CanonicalChunk] = []
|
||||
for document in documents:
|
||||
|
||||
Reference in New Issue
Block a user