feat: index semantic records in qdrant
This commit is contained in:
@@ -10,17 +10,18 @@ from pathlib import Path
|
||||
import typer
|
||||
from sqlalchemy.exc import OperationalError, ProgrammingError
|
||||
|
||||
from tht.cli.config_cmd import CONFIG_OPT
|
||||
from tht.cli.schema_cmd import _load_config_or_exit
|
||||
from tht.cli._guards import (
|
||||
has_vector_write_rest,
|
||||
require_server_profile,
|
||||
require_vector_write_allowed,
|
||||
)
|
||||
from tht.cli.config_cmd import CONFIG_OPT
|
||||
from tht.cli.schema_cmd import _load_config_or_exit
|
||||
from tht.cli.session_cmd import load_snapshot_or_exit
|
||||
from tht.cli.vector_cmd import require_vector_cfg
|
||||
|
||||
memory_app = typer.Typer(help="Review memory (registro canonico + indice pgvector)")
|
||||
DECISION_OPT = typer.Option(None, "--decision", help="Seq da promuovere (ripetibile).")
|
||||
|
||||
|
||||
def registry_path(cfg) -> Path:
|
||||
@@ -29,18 +30,23 @@ def registry_path(cfg) -> Path:
|
||||
|
||||
def _resync_memory(cfg):
|
||||
"""Risincronizza l'indice pgvector col registro corrente (incrementale)."""
|
||||
from tht.cli.vector_cmd import make_embedder, open_store
|
||||
from tht.adapters.factory import build_vector_store
|
||||
from tht.cli.vector_cmd import make_embedder, sync_canonical_records
|
||||
from tht.memory import load_registry, memory_vector_records
|
||||
|
||||
records = memory_vector_records(load_registry(registry_path(cfg)))
|
||||
store = open_store(cfg, "memory")
|
||||
return store.sync(records, make_embedder(cfg.embeddings), kinds={"memory"})
|
||||
return sync_canonical_records(
|
||||
"memory",
|
||||
records,
|
||||
store=build_vector_store(cfg, require_write=True),
|
||||
embedder=make_embedder(cfg.embeddings),
|
||||
)
|
||||
|
||||
|
||||
@memory_app.command("promote")
|
||||
def promote_cmd(
|
||||
session: str = typer.Option(..., "--session"),
|
||||
decision: list[int] = typer.Option(None, "--decision", help="Seq da promuovere (ripetibile)."),
|
||||
decision: list[int] = DECISION_OPT,
|
||||
preview: bool = typer.Option(False, "--preview", help="Mostra i candidati in JSON, non scrive."),
|
||||
json_out: bool = typer.Option(False, "--json", help="Output JSON (per Pi)."),
|
||||
config: Path = CONFIG_OPT,
|
||||
@@ -55,7 +61,9 @@ def promote_cmd(
|
||||
|
||||
if preview:
|
||||
from tht.memory import (
|
||||
MAX_PROMOTION_CANDIDATES, preview_promotions_snapshot, reusable_promotions_snapshot,
|
||||
MAX_PROMOTION_CANDIDATES,
|
||||
preview_promotions_snapshot,
|
||||
reusable_promotions_snapshot,
|
||||
)
|
||||
cand = preview_promotions_snapshot(snapshot, registry_path(cfg))
|
||||
extra = len(reusable_promotions_snapshot(snapshot, registry_path(cfg))) - len(cand)
|
||||
@@ -304,8 +312,8 @@ def update_cmd(
|
||||
"""Modifica i campi di merito di una memoria (provenienza immutabile)."""
|
||||
from typing import get_args
|
||||
|
||||
from tht.memory import MemoryNotFound, update_record
|
||||
from tht.decisions import DecisionType
|
||||
from tht.memory import MemoryNotFound, update_record
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
|
||||
|
||||
@@ -2,9 +2,15 @@ from pathlib import Path
|
||||
|
||||
import typer
|
||||
|
||||
from tht.cli._guards import has_vector_write_rest, require_server_profile, require_vector_write_allowed
|
||||
from tht.cli._guards import (
|
||||
has_vector_write_rest,
|
||||
require_server_profile,
|
||||
require_vector_write_allowed,
|
||||
)
|
||||
from tht.cli.config_cmd import CONFIG_OPT
|
||||
from tht.cli.schema_cmd import _load_config_or_exit, annotations_path, physical_path
|
||||
from tht.ports.vector import VectorWriteRecord
|
||||
from tht.vectorstore.store import SyncStats, content_hash
|
||||
|
||||
vector_app = typer.Typer(help="Indice semantico pgvector (derivato, rigenerabile)")
|
||||
|
||||
@@ -69,6 +75,31 @@ def open_searcher(cfg):
|
||||
return AdapterSearcher()
|
||||
|
||||
|
||||
def sync_canonical_records(collection, records, *, store, embedder):
|
||||
kinds = sorted({record.kind for record in records})
|
||||
existing = store.existing_hashes(collection, kinds)
|
||||
pending = []
|
||||
stats = SyncStats()
|
||||
changed = []
|
||||
for record in records:
|
||||
hashed = content_hash(record.content)
|
||||
current = existing.get(record.id)
|
||||
if current == hashed:
|
||||
stats.unchanged += 1
|
||||
continue
|
||||
changed.append((record, hashed, current is None))
|
||||
if changed:
|
||||
embeddings = embedder.embed_documents([record.content for record, *_ in changed])
|
||||
for (record, hashed, is_added), embedding in zip(changed, embeddings, strict=True):
|
||||
pending.append(VectorWriteRecord(record=record, embedding=embedding, content_hash=hashed))
|
||||
if is_added:
|
||||
stats.added += 1
|
||||
else:
|
||||
stats.updated += 1
|
||||
store.upsert(collection, pending)
|
||||
return stats
|
||||
|
||||
|
||||
def _print_stats(stats) -> None:
|
||||
typer.secho(
|
||||
f"OK: {stats.added} nuovi, {stats.updated} aggiornati, "
|
||||
@@ -88,7 +119,6 @@ def init_cmd(
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from tht.vectorstore.embeddings import EmbeddingsError
|
||||
|
||||
from tht.vectorstore.reader import ALL_TABLES
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
@@ -131,8 +161,12 @@ def index_schema_cmd(config: Path = CONFIG_OPT) -> None:
|
||||
physical = PhysicalSchema.from_yaml(phys_file)
|
||||
annotations = Annotations.from_yaml(annotations_path(cfg))
|
||||
records = schema_records(physical, annotations)
|
||||
store = open_store(cfg, "schema_records")
|
||||
stats = store.sync(
|
||||
records, make_embedder(cfg.embeddings), kinds={"schema_table", "schema_column"}
|
||||
from tht.adapters.factory import build_vector_store
|
||||
|
||||
stats = sync_canonical_records(
|
||||
"schema_records",
|
||||
records,
|
||||
store=build_vector_store(cfg, require_write=True),
|
||||
embedder=make_embedder(cfg.embeddings),
|
||||
)
|
||||
_print_stats(stats)
|
||||
|
||||
Reference in New Issue
Block a user