feat(evidence): use server-side Qdrant BM25 retrieval
This commit is contained in:
@@ -14,6 +14,14 @@ from tht.cli.config_cmd import CONFIG_OPT
|
||||
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
|
||||
|
||||
|
||||
def _bm25_language(workspace_language: str) -> str:
|
||||
languages = {"en": "english", "it": "italian"}
|
||||
try:
|
||||
return languages[workspace_language]
|
||||
except KeyError as exc:
|
||||
raise ValueError("workspace language is unsupported for Qdrant BM25") from exc
|
||||
|
||||
|
||||
def _evidence_json_context(config: Path):
|
||||
from tht.cli.schema_cmd import _load_config_or_exit
|
||||
|
||||
@@ -112,6 +120,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
||||
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
|
||||
pipeline_version="evidence-v1",
|
||||
retain_published_generations=cfg.vector.retain_published_generations,
|
||||
sparse_language=_bm25_language(cfg.language),
|
||||
)
|
||||
def fingerprint(value: str) -> str:
|
||||
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
|
||||
@@ -145,6 +154,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
|
||||
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
|
||||
pipeline_version="evidence-v1",
|
||||
retain_published_generations=cfg.vector.retain_published_generations,
|
||||
sparse_language=_bm25_language(cfg.language),
|
||||
)
|
||||
pipeline.workspace_id = cfg._workspace_id
|
||||
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
|
||||
|
||||
Reference in New Issue
Block a user