feat: add P2 harness machine contracts
This commit is contained in:
@@ -1,17 +1,18 @@
|
||||
"""One-shot preprocessing commands."""
|
||||
# ruff: noqa: BLE001
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
|
||||
from tht.cli.config_cmd import CONFIG_OPT
|
||||
from tht.config import workspace_id_from_path
|
||||
|
||||
from tht.cli.schema_cmd import _load_config_or_exit
|
||||
from tht.config import workspace_id_for_config
|
||||
|
||||
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
|
||||
|
||||
@@ -22,7 +23,8 @@ def run_dwh_from_config(
|
||||
from tht.cli.lsh_cmd import build_lsh_artifacts
|
||||
from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog
|
||||
from tht.jobs.dwh_pipeline import (
|
||||
DwhPreprocessPipeline, config_dwh_binding,
|
||||
DwhPreprocessPipeline,
|
||||
config_dwh_binding,
|
||||
)
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
@@ -64,7 +66,6 @@ def _parse_dwh_steps(value: str) -> tuple[str, ...]:
|
||||
|
||||
def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = None):
|
||||
from tht.adapters.factory import build_evidence_sources, build_vector_store
|
||||
from tht.cli.schema_cmd import _load_config_or_exit
|
||||
from tht.cli.vector_cmd import make_embedder
|
||||
from tht.corpus.chunk import ChunkPolicy
|
||||
from tht.corpus.pipeline import CorpusPipeline
|
||||
@@ -87,7 +88,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
||||
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
return pipeline.run_as_job(
|
||||
workspace_id=workspace_id_from_path(config),
|
||||
workspace_id=workspace_id_for_config(cfg, config),
|
||||
workspace_root=corpus_root.parent,
|
||||
config_fingerprint=fingerprint(cfg.model_dump_json()),
|
||||
input_fingerprint=fingerprint(config.resolve().as_posix()),
|
||||
@@ -116,7 +117,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
|
||||
pipeline_version="evidence-v1",
|
||||
retain_published_generations=cfg.vector.retain_published_generations,
|
||||
)
|
||||
pipeline.workspace_id = workspace_id_from_path(config)
|
||||
pipeline.workspace_id = workspace_id_for_config(cfg, config)
|
||||
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user