mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor(evidence): migrate acquisition and preprocessing (#30)

Browse Source
This commit is contained in:
marcopan
2026-08-24 02:25:11 +02:00
parent d5e78febd3
commit 44f1efa5ba
18 changed files with 554 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/cli/preprocess_cmd.py
+8 -8
View File
@@ -93,19 +93,19 @@ def _parse_dwh_steps(value: str) -> tuple[str, ...]:
def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = None):
from tht.adapters.factory import build_evidence_sources, build_vector_store
from tht.adapters.factory import build_vector_store
from tht.cli.schema_cmd import _load_config_or_exit
from tht.cli.vector_cmd import make_embedder
from tht.corpus.chunk import ChunkPolicy
from tht.corpus.pipeline import CorpusPipeline
from tht.corpus.store import CorpusStore
from tht.evidence import build_preprocessing_pipeline, build_sources
cfg = _load_config_or_exit(config)
if cfg.embeddings is None:
raise RuntimeError("embeddings are not configured")
corpus_root = cfg.paths.artifacts.parent / "corpus"
pipeline = CorpusPipeline(
store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg),
pipeline = build_preprocessing_pipeline(
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
embedder=make_embedder(cfg.embeddings),
vector_store=build_vector_store(cfg, require_write=True),
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
@@ -127,19 +127,19 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
def gc_from_config(config: Path, *, dry_run: bool = False):
from tht.adapters.factory import build_evidence_sources, build_vector_store
from tht.adapters.factory import build_vector_store
from tht.cli.schema_cmd import _load_config_or_exit
from tht.cli.vector_cmd import make_embedder
from tht.corpus.chunk import ChunkPolicy
from tht.corpus.pipeline import CorpusPipeline
from tht.corpus.store import CorpusStore
from tht.evidence import build_preprocessing_pipeline, build_sources
cfg = _load_config_or_exit(config)
if cfg.embeddings is None:
raise RuntimeError("embeddings are not configured")
corpus_root = cfg.paths.artifacts.parent / "corpus"
pipeline = CorpusPipeline(
store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg),
pipeline = build_preprocessing_pipeline(
store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True),
embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),