feat(evidence): add safe retention and materialized reads

2026-07-12 05:02:47 +02:00
parent b6a52995ae
commit f3b49f41c8
11 changed files with 310 additions and 14 deletions
@@ -34,6 +34,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
        embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
        chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
        pipeline_version="evidence-v1",
+        retain_published_generations=cfg.vector.retain_published_generations,
    )
    def fingerprint(value: str) -> str:
        return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
@@ -48,13 +49,55 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
    )


+def gc_from_config(config: Path, *, dry_run: bool = False):
+    from tht.adapters.factory import build_evidence_sources, build_vector_store
+    from tht.cli.schema_cmd import _load_config_or_exit
+    from tht.cli.vector_cmd import make_embedder
+    from tht.corpus.chunk import ChunkPolicy
+    from tht.corpus.pipeline import CorpusPipeline
+    from tht.corpus.store import CorpusStore
+
+    cfg = _load_config_or_exit(config)
+    if cfg.embeddings is None:
+        raise RuntimeError("embeddings are not configured")
+    corpus_root = cfg.paths.artifacts.parent / "corpus"
+    pipeline = CorpusPipeline(
+        store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg),
+        embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True),
+        embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
+        chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
+        pipeline_version="evidence-v1",
+        retain_published_generations=cfg.vector.retain_published_generations,
+    )
+    with pipeline.store.writer_lock():
+        return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
+
+
@preprocess_app.command("evidence")
 def evidence_cmd(
+    action: str | None = typer.Argument(None),
    config: Path = CONFIG_OPT,
    dry_run: bool = typer.Option(False, "--dry-run"),
    resume: str | None = typer.Option(None, "--resume"),
    json_output: bool = typer.Option(False, "--json"),
 ) -> None:
+    if action is not None and action != "gc":
+        raise typer.BadParameter("only the optional 'gc' action is supported")
+    if action == "gc":
+        try:
+            payload = gc_from_config(config, dry_run=dry_run)
+        except Exception:
+            payload = {"status": "failed", "error": "evidence cleanup failed"}
+            if json_output:
+                typer.echo(json.dumps(payload, sort_keys=True))
+            else:
+                typer.secho("ERRORE: evidence cleanup failed", fg=typer.colors.RED, err=True)
+            raise typer.Exit(code=1) from None
+        if json_output:
+            typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
+        else:
+            typer.echo(f"OK: evicted={len(payload['evicted'])} failures={len(payload['failures'])}")
+        return
    if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None:
        payload = {"status": "failed", "error": "resume requires a preprocessing run id"}
        if json_output: