ThothII/harness/tht/cli/vector_cmd.py

import hashlib
import json
from pathlib import Path

import typer

from tht.cli._guards import require_vector_write_allowed
from tht.cli.config_cmd import CONFIG_OPT
from tht.cli.schema_cmd import _load_config_or_exit, annotations_path, physical_path
from tht.ports.vector import VectorWriteRecord
from tht.vectorstore.store import SyncStats, content_hash

vector_app = typer.Typer(help="Indice semantico Qdrant (derivato, rigenerabile)")


def _artifact_digest(path: Path) -> str:
    try:
        contents = path.read_bytes()
    except FileNotFoundError:
        # An absent annotations file is valid (empty curation); digest the empty artifact.
        contents = b""
    return "sha256:" + hashlib.sha256(contents).hexdigest()


def _emit_json(payload: dict) -> None:
    typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))


def make_embedder(embeddings_cfg):
    """Factory del client embeddings (monkeypatchabile nei test)."""
    from tht.vectorstore.embeddings import OllamaEmbeddings

    return OllamaEmbeddings(embeddings_cfg)


def require_vector_cfg(cfg):
    missing = []
    if cfg.embeddings is None:
        missing.append("embeddings")
    if cfg.vectors is None:
        missing.append("vectors")
    if missing:
        typer.secho(
            f"ERRORE: sezioni mancanti nel workspace yaml: {', '.join(missing)}.",
            fg=typer.colors.RED, err=True,
        )
        raise typer.Exit(code=1)


def open_searcher(cfg):
    """Searcher per la lettura semantic search sul runtime vettoriale attivo."""
    from tht.adapters.factory import build_vector_store
    from tht.vectorstore.reader import tables_for_kinds

    store = build_vector_store(cfg)

    class AdapterSearcher:
        def search(self, query_vec, top_n=10, kinds=None, metadata_filter=None):
            return store.search(
                tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds,
                metadata_filter=metadata_filter,
            )

    return AdapterSearcher()


def sync_canonical_records(collection, records, *, store, embedder):
    kinds = sorted({record.kind for record in records})
    existing = store.existing_hashes(collection, kinds)
    pending = []
    stats = SyncStats()
    changed = []
    for record in records:
        hashed = content_hash(record.content)
        current = existing.get(record.id)
        if current == hashed:
            stats.unchanged += 1
            continue
        changed.append((record, hashed, current is None))
    if changed:
        embeddings = embedder.embed_documents([record.content for record, *_ in changed])
        for (record, hashed, is_added), embedding in zip(changed, embeddings, strict=True):
            pending.append(VectorWriteRecord(record=record, embedding=embedding, content_hash=hashed))
            if is_added:
                stats.added += 1
            else:
                stats.updated += 1
        store.upsert(collection, pending)
    return stats


def _print_stats(stats) -> None:
    typer.secho(
        f"OK: {stats.added} nuovi, {stats.updated} aggiornati, "
        f"{stats.deleted} rimossi, {stats.unchanged} invariati",
        fg=typer.colors.GREEN,
    )


@vector_app.command("index-schema")
def index_schema_cmd(
    config: Path = CONFIG_OPT,
    json_output: bool = typer.Option(False, "--json"),
) -> None:
    """Embedda e sincronizza i record schema (tabelle e colonne) nel semantic store."""
    from tht.adapters.factory import build_vector_store
    from tht.mschema.models import Annotations, PhysicalSchema
    from tht.ports.vector import VectorStoreError
    from tht.vectorstore.records import schema_records

    cfg = _load_config_or_exit(config)
    require_vector_write_allowed(cfg, "vector index-schema")
    require_vector_cfg(cfg)
    phys_file = physical_path(cfg)
    if not phys_file.exists():
        message = f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`."
        if json_output:
            _emit_json({
                "code": "schema_missing",
                "error": "physical schema is missing",
                "operation": "index_schema",
                "schemaVersion": 1,
                "status": "failed",
                "workspaceId": cfg._workspace_id,
                "workspaceRevision": cfg._workspace_revision,
            })
        else:
            typer.secho(message, fg=typer.colors.RED, err=True)
        raise typer.Exit(code=1)
    physical = PhysicalSchema.from_yaml(phys_file)
    annotations_file = annotations_path(cfg)
    annotations = Annotations.from_yaml(annotations_file)
    records = schema_records(physical, annotations)
    try:
        stats = sync_canonical_records(
            "schema_records",
            records,
            store=build_vector_store(cfg, require_write=True),
            embedder=make_embedder(cfg.embeddings),
        )
    except VectorStoreError as exc:
        code = str(exc)
        error = "semantic index incompatible" if code == "semantic_index_incompatible" else "schema indexing failed"
        if json_output:
            _emit_json({
                "code": code,
                "error": error,
                "operation": "index_schema",
                "schemaVersion": 1,
                "status": "failed",
                "workspaceId": cfg._workspace_id,
                "workspaceRevision": cfg._workspace_revision,
            })
        else:
            typer.secho(f"ERRORE: {error}", fg=typer.colors.RED, err=True)
        raise typer.Exit(code=1) from None
    if json_output:
        _emit_json({
            "artifactIdentities": [
                {"digest": _artifact_digest(annotations_file), "kind": "schema_annotations"},
                {"digest": _artifact_digest(phys_file), "kind": "physical_schema"},
            ],
            "code": "ok",
            "collection": cfg.vectors.collection,
            "counts": {
                "added": stats.added,
                "columns": sum(len(table.columns) for table in physical.tables.values()),
                "deleted": stats.deleted,
                "records": len(records),
                "tables": len(physical.tables),
                "unchanged": stats.unchanged,
                "updated": stats.updated,
            },
            "operation": "index_schema",
            "schemaVersion": 1,
            "status": "succeeded",
            "workspaceId": cfg._workspace_id,
            "workspaceRevision": cfg._workspace_revision,
        })
        return
    _print_stats(stats)