feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s
Publish documentation / publish (push) Successful in 2m12s
This commit is contained in:
@@ -6,7 +6,7 @@ import typer
|
||||
|
||||
from tht.cli._guards import require_vector_write_allowed
|
||||
from tht.cli.config_cmd import CONFIG_OPT
|
||||
from tht.cli.schema_cmd import _load_config_or_exit, annotations_path, physical_path
|
||||
from tht.cli.schema_cmd import _load_config_or_exit
|
||||
from tht.ports.vector import VectorWriteRecord
|
||||
from tht.vectorstore.store import SyncStats, content_hash
|
||||
|
||||
@@ -112,42 +112,14 @@ def index_schema_cmd(
|
||||
config: Path = CONFIG_OPT,
|
||||
json_output: bool = typer.Option(False, "--json"),
|
||||
) -> None:
|
||||
"""Embedda e sincronizza i record schema (tabelle e colonne) nel semantic store."""
|
||||
from tht.adapters.factory import build_vector_store
|
||||
from tht.mschema.models import Annotations, PhysicalSchema
|
||||
"""Replace the schema slice from the PostgreSQL Catalog projection."""
|
||||
from tht.ports.vector import VectorStoreError
|
||||
from tht.vectorstore.records import schema_records
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
require_vector_write_allowed(cfg, "vector index-schema")
|
||||
require_vector_cfg(cfg)
|
||||
phys_file = physical_path(cfg)
|
||||
if not phys_file.exists():
|
||||
message = f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`."
|
||||
if json_output:
|
||||
_emit_json({
|
||||
"code": "schema_missing",
|
||||
"error": "physical schema is missing",
|
||||
"operation": "index_schema",
|
||||
"schemaVersion": 1,
|
||||
"status": "failed",
|
||||
"workspaceId": cfg._workspace_id,
|
||||
"workspaceRevision": cfg._workspace_revision,
|
||||
})
|
||||
else:
|
||||
typer.secho(message, fg=typer.colors.RED, err=True)
|
||||
raise typer.Exit(code=1)
|
||||
physical = PhysicalSchema.from_yaml(phys_file)
|
||||
annotations_file = annotations_path(cfg)
|
||||
annotations = Annotations.from_yaml(annotations_file)
|
||||
records = schema_records(physical, annotations)
|
||||
try:
|
||||
stats = sync_canonical_records(
|
||||
"schema_records",
|
||||
records,
|
||||
store=build_vector_store(cfg, require_write=True),
|
||||
embedder=make_embedder(cfg.embeddings),
|
||||
)
|
||||
stats, counts, snapshot_path = index_catalog_schema(cfg)
|
||||
except VectorStoreError as exc:
|
||||
code = str(exc)
|
||||
error = "semantic index incompatible" if code == "semantic_index_incompatible" else "schema indexing failed"
|
||||
@@ -167,17 +139,17 @@ def index_schema_cmd(
|
||||
if json_output:
|
||||
_emit_json({
|
||||
"artifactIdentities": [
|
||||
{"digest": _artifact_digest(annotations_file), "kind": "schema_annotations"},
|
||||
{"digest": _artifact_digest(phys_file), "kind": "physical_schema"},
|
||||
{"digest": _artifact_digest(snapshot_path), "kind": "catalog_metadata_snapshot"},
|
||||
],
|
||||
"code": "ok",
|
||||
"collection": cfg.vectors.collection,
|
||||
"collection": cfg.vectors.collections["reference"],
|
||||
"counts": {
|
||||
"added": stats.added,
|
||||
"columns": sum(len(table.columns) for table in physical.tables.values()),
|
||||
"columns": counts["columns"],
|
||||
"deleted": stats.deleted,
|
||||
"records": len(records),
|
||||
"tables": len(physical.tables),
|
||||
"records": counts["records"],
|
||||
"relationships": counts["relationships"],
|
||||
"tables": counts["tables"],
|
||||
"unchanged": stats.unchanged,
|
||||
"updated": stats.updated,
|
||||
},
|
||||
@@ -189,3 +161,32 @@ def index_schema_cmd(
|
||||
})
|
||||
return
|
||||
_print_stats(stats)
|
||||
|
||||
|
||||
def index_catalog_schema(cfg):
|
||||
from tht.adapters.factory import build_vector_store
|
||||
from tht.mschema.catalog_snapshot import load_catalog_metadata_snapshot
|
||||
from tht.vectorstore.records import catalog_schema_records
|
||||
|
||||
snapshot_path = cfg.paths.catalog_metadata_snapshot
|
||||
if snapshot_path is None:
|
||||
raise ValueError("catalog metadata snapshot is not configured")
|
||||
snapshot = load_catalog_metadata_snapshot(snapshot_path, cfg._workspace_id)
|
||||
records = catalog_schema_records(snapshot)
|
||||
store = build_vector_store(cfg, require_write=True)
|
||||
deleted = store.delete_kinds(
|
||||
"schema_records", ["schema_table", "schema_column", "schema_relationship"]
|
||||
)
|
||||
stats = sync_canonical_records(
|
||||
"schema_records",
|
||||
records,
|
||||
store=store,
|
||||
embedder=make_embedder(cfg.embeddings),
|
||||
)
|
||||
stats.deleted = deleted
|
||||
return stats, {
|
||||
"tables": len(snapshot.tables),
|
||||
"columns": sum(len(table.columns) for table in snapshot.tables),
|
||||
"relationships": len(snapshot.relationships),
|
||||
"records": len(records),
|
||||
}, snapshot_path
|
||||
|
||||
Reference in New Issue
Block a user