refactor(harness): renaming prodotto tht (Onda -1)
Thoth (tht) è il prodotto, PSD è il cliente. Nessun riferimento al contesto
clinico nel codice.
Rinomine:
- comando+package nsp→tht (dir nsp/→tht/, 46 import, pyproject entry point)
- gate nsp-gate.js→tht-gate.js (+ rewrite token, relayIfNspFails→relayIfThtFails)
- workspace chirone.{example,test}.yaml→tht.{example,test}.yaml (generici)
- env THOTH_→THT_ (19 var) + NSP_ stragglers (NSP_HARNESS_ROOT, NSP_SESSION)
- commenti/docstring chirone/psdwp3/policlinico neutralizzati ('the reference
implementation', 'the DWH')
Aggiunto [tool.setuptools.packages.find] include=['tht*'] (necessario: l'auto-
discovery rompeva con tht/ + workspaces/ come top-level multipli).
.env operatore aggiornato in-place (prefissi THT_, valori preservati, gitignored).
Verifica: pytest 109 passed, npm test 14 pass, tht phase meta --json OK, zero
residui nsp/THOTH_/NSP_/chirone nel package.
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
import requests
|
||||
|
||||
from tht.config import EmbeddingsConfig
|
||||
|
||||
DOC_PREFIX = "search_document: "
|
||||
QUERY_PREFIX = "search_query: "
|
||||
|
||||
|
||||
class EmbeddingsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class OllamaEmbeddings:
|
||||
"""Client embeddings via Ollama. Applica i prefissi di task richiesti da nomic v2:
|
||||
ometterli degrada il retrieval in modo silenzioso."""
|
||||
|
||||
def __init__(self, cfg: EmbeddingsConfig):
|
||||
self.cfg = cfg
|
||||
|
||||
def _embed(self, texts: list[str]) -> list[list[float]]:
|
||||
url = f"{self.cfg.base_url.rstrip('/')}/api/embed"
|
||||
out: list[list[float]] = []
|
||||
for i in range(0, len(texts), self.cfg.batch_size):
|
||||
batch = texts[i : i + self.cfg.batch_size]
|
||||
try:
|
||||
resp = requests.post(
|
||||
url, json={"model": self.cfg.model, "input": batch},
|
||||
timeout=self.cfg.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
raise EmbeddingsError(
|
||||
f"Ollama non raggiungibile su {self.cfg.base_url} "
|
||||
f"(modello {self.cfg.model}): {e}"
|
||||
) from e
|
||||
embeddings = resp.json().get("embeddings", [])
|
||||
for v in embeddings:
|
||||
if len(v) != self.cfg.dim:
|
||||
raise EmbeddingsError(
|
||||
f"dimensione embedding inattesa: {len(v)} != {self.cfg.dim} "
|
||||
f"(modello {self.cfg.model})"
|
||||
)
|
||||
out.extend(embeddings)
|
||||
return out
|
||||
|
||||
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||
return self._embed([DOC_PREFIX + t for t in texts])
|
||||
|
||||
def embed_query(self, text: str) -> list[float]:
|
||||
return self._embed([QUERY_PREFIX + text])[0]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Lettura del pgvector dietro un'unica interfaccia `.search(query_vec, top_n, kinds)`, così
|
||||
`search.combined_search` resta agnostico al transport. Due implementazioni:
|
||||
|
||||
- `RestSearcher` → produzione: similarity search via REST (`search_similar`).
|
||||
- `DirectSearcher` → dev/test: connessione diretta a Postgres/pgvector.
|
||||
|
||||
Entrambe mappano i `kind` sulle tabelle per-dominio dello schema `vectors`.
|
||||
"""
|
||||
|
||||
from sqlalchemy import Engine
|
||||
|
||||
from tht.vectorstore.rest_client import VectorRestClient
|
||||
from tht.vectorstore.store import VectorHit, VectorStore, hit_from_metadata
|
||||
|
||||
# kind Thoth → tabella dello schema `vectors`.
|
||||
KIND_TO_TABLE = {
|
||||
"schema_table": "schema_records",
|
||||
"schema_column": "schema_records",
|
||||
"evidence": "evidence",
|
||||
"memory": "memory",
|
||||
}
|
||||
ALL_TABLES = ["schema_records", "evidence", "memory"]
|
||||
|
||||
|
||||
def tables_for_kinds(kinds: list[str] | None) -> list[str]:
|
||||
"""Tabelle da interrogare per i kind richiesti (tutte se kinds è vuoto/None)."""
|
||||
if not kinds:
|
||||
return list(ALL_TABLES)
|
||||
return sorted({KIND_TO_TABLE[k] for k in kinds if k in KIND_TO_TABLE})
|
||||
|
||||
|
||||
def _merge(hits: list[VectorHit], top_n: int) -> list[VectorHit]:
|
||||
return sorted(hits, key=lambda h: h.similarity, reverse=True)[:top_n]
|
||||
|
||||
|
||||
class RestSearcher:
|
||||
"""Similarity search via REST: una chiamata `search_similar` per tabella, poi fusione."""
|
||||
|
||||
def __init__(self, client: VectorRestClient):
|
||||
self.client = client
|
||||
|
||||
def search(
|
||||
self, query_vec: list[float], top_n: int = 10, kinds: list[str] | None = None
|
||||
) -> list[VectorHit]:
|
||||
hits: list[VectorHit] = []
|
||||
for table in tables_for_kinds(kinds):
|
||||
for row in self.client.search_similar(table, query_vec, top_n):
|
||||
hits.append(hit_from_metadata(row.get("similarity", 0.0), row.get("metadata")))
|
||||
return _merge(hits, top_n)
|
||||
|
||||
|
||||
class DirectSearcher:
|
||||
"""Similarity search diretta su Postgres/pgvector, interrogando le tabelle per-dominio."""
|
||||
|
||||
def __init__(self, engine: Engine, schema: str = "vectors", dim: int = 768):
|
||||
self.engine = engine
|
||||
self.schema = schema
|
||||
self.dim = dim
|
||||
|
||||
def search(
|
||||
self, query_vec: list[float], top_n: int = 10, kinds: list[str] | None = None
|
||||
) -> list[VectorHit]:
|
||||
hits: list[VectorHit] = []
|
||||
for table in tables_for_kinds(kinds):
|
||||
store = VectorStore(self.engine, schema=self.schema, table=table, dim=self.dim)
|
||||
hits.extend(store.search(query_vec, top_n=top_n))
|
||||
return _merge(hits, top_n)
|
||||
@@ -0,0 +1,101 @@
|
||||
import re
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from tht.evidence.model import EvidenceDoc
|
||||
from tht.mschema.models import Annotations, PhysicalSchema
|
||||
|
||||
MAX_EXAMPLES_IN_RECORD = 5
|
||||
|
||||
|
||||
class VectorRecord(BaseModel):
|
||||
id: str
|
||||
kind: str # evidence | schema_table | schema_column
|
||||
ref: str # file/chiave canonica di provenienza
|
||||
title: str
|
||||
content: str
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
def split_markdown(text: str, max_chars: int) -> list[str]:
|
||||
"""Spezza un markdown: intero se sta nel limite, altrimenti per heading '##',
|
||||
e in ultima istanza per accumulo greedy di righe."""
|
||||
if len(text) <= max_chars:
|
||||
return [text]
|
||||
parts = re.split(r"(?=^## )", text, flags=re.MULTILINE)
|
||||
chunks: list[str] = []
|
||||
for part in parts:
|
||||
part = part.strip("\n")
|
||||
if not part:
|
||||
continue
|
||||
if len(part) <= max_chars:
|
||||
chunks.append(part)
|
||||
continue
|
||||
current: list[str] = []
|
||||
size = 0
|
||||
for line in part.splitlines():
|
||||
if size + len(line) > max_chars and current:
|
||||
chunks.append("\n".join(current))
|
||||
current, size = [], 0
|
||||
current.append(line)
|
||||
size += len(line) + 1
|
||||
if current:
|
||||
chunks.append("\n".join(current))
|
||||
return chunks
|
||||
|
||||
|
||||
def evidence_records(docs: list[EvidenceDoc], max_chunk_chars: int) -> list[VectorRecord]:
|
||||
"""Record per tutte le evidence presenti: la sola presenza basta a indicizzarle."""
|
||||
records: list[VectorRecord] = []
|
||||
for doc in docs:
|
||||
content = f"{doc.title}\n\n{doc.body}"
|
||||
for i, chunk in enumerate(split_markdown(content, max_chunk_chars)):
|
||||
records.append(
|
||||
VectorRecord(
|
||||
id=f"evidence:{doc.id}:{i}",
|
||||
kind="evidence",
|
||||
ref=str(doc.path) if doc.path else doc.id,
|
||||
title=doc.title,
|
||||
content=chunk,
|
||||
metadata={
|
||||
"status": doc.status, "tier": doc.tier,
|
||||
"tables": doc.tables, "concepts": doc.concepts,
|
||||
},
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def schema_records(physical: PhysicalSchema, annotations: Annotations) -> list[VectorRecord]:
|
||||
"""Un record per tabella e uno per colonna, da mschema (physical + annotations)."""
|
||||
records: list[VectorRecord] = []
|
||||
for table_name, table in physical.tables.items():
|
||||
ann_t = annotations.tables.get(table_name)
|
||||
t_desc = (ann_t.description if ann_t and ann_t.description else table.comment)
|
||||
t_concepts = ann_t.concepts if ann_t else []
|
||||
lines = [f"Tabella {table_name}", t_desc]
|
||||
if t_concepts:
|
||||
lines.append("Concetti: " + ", ".join(t_concepts))
|
||||
lines.append("Colonne: " + ", ".join(table.columns))
|
||||
records.append(
|
||||
VectorRecord(
|
||||
id=f"schema_table:{table_name}", kind="schema_table", ref=table_name,
|
||||
title=table_name, content="\n".join(filter(None, lines)),
|
||||
)
|
||||
)
|
||||
for column_name, column in table.columns.items():
|
||||
ann_c = ann_t.columns.get(column_name) if ann_t else None
|
||||
c_desc = (ann_c.description if ann_c and ann_c.description else column.comment)
|
||||
lines = [f"Colonna {table_name}.{column_name} ({column.type})", c_desc]
|
||||
if ann_c and ann_c.synonyms:
|
||||
lines.append("Sinonimi: " + ", ".join(ann_c.synonyms))
|
||||
if column.examples:
|
||||
lines.append("Esempi: " + ", ".join(column.examples[:MAX_EXAMPLES_IN_RECORD]))
|
||||
records.append(
|
||||
VectorRecord(
|
||||
id=f"schema_column:{table_name}.{column_name}", kind="schema_column",
|
||||
ref=f"{table_name}.{column_name}", title=f"{table_name}.{column_name}",
|
||||
content="\n".join(filter(None, lines)),
|
||||
)
|
||||
)
|
||||
return records
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Client per la similarity search del pgvector esposta via Supabase/PostgREST.
|
||||
|
||||
Endpoint dedicato (es. https://host/vector/v1/), distinto dal DWH. La lettura usa
|
||||
`search_similar`; la scrittura remota usa RPC allowlist con una API key separata.
|
||||
Errori in italiano e azionabili, stile `rest/client.py`.
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
from tht.config import RestConfig
|
||||
|
||||
|
||||
class VectorRestError(Exception):
|
||||
"""Errore di accesso al vector store via REST, con messaggio leggibile per il reviewer."""
|
||||
|
||||
|
||||
class VectorRestClient:
|
||||
def __init__(self, cfg: RestConfig):
|
||||
self.cfg = cfg
|
||||
self._base = cfg.base_url.rstrip("/")
|
||||
|
||||
@property
|
||||
def api_key(self) -> str:
|
||||
"""The REST API key for this client (spec D11: reader and writer carry
|
||||
distinct keys against the same endpoint)."""
|
||||
return self.cfg.api_key
|
||||
|
||||
def _post(self, fn: str, args: dict) -> requests.Response:
|
||||
url = f"{self._base}/rpc/{fn}"
|
||||
verify: bool | str = self.cfg.ssl_ca if self.cfg.ssl_ca else True
|
||||
try:
|
||||
return requests.post(
|
||||
url,
|
||||
json=args,
|
||||
headers={"X-API-Key": self.cfg.api_key},
|
||||
timeout=self.cfg.timeout,
|
||||
verify=verify,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
raise VectorRestError(
|
||||
f"Vector REST non raggiungibile su {self.cfg.base_url} (rpc {fn}): {e}"
|
||||
) from e
|
||||
|
||||
def _error_msg(self, fn: str, resp: requests.Response) -> str:
|
||||
try:
|
||||
body = resp.json()
|
||||
detail = body.get("message") or body.get("details") or resp.text
|
||||
except Exception:
|
||||
detail = resp.text
|
||||
return f"Vector REST rpc {fn} → HTTP {resp.status_code}: {detail}"
|
||||
|
||||
def _call(self, fn: str, args: dict):
|
||||
resp = self._post(fn, args)
|
||||
if not resp.ok:
|
||||
raise VectorRestError(self._error_msg(fn, resp))
|
||||
if resp.status_code == 204 or not resp.text:
|
||||
return None
|
||||
return resp.json()
|
||||
|
||||
def search_similar(
|
||||
self, table_name: str, query_embedding: list[float], limit_count: int
|
||||
) -> list[dict]:
|
||||
"""Ricerca per similarità coseno su `vectors.<table_name>`: ritorna le righe
|
||||
`{id, similarity, metadata}` ordinate per similarity decrescente."""
|
||||
return self._call(
|
||||
"search_similar",
|
||||
{
|
||||
"query_embedding": query_embedding,
|
||||
"limit_count": limit_count,
|
||||
"table_name": table_name,
|
||||
},
|
||||
) or []
|
||||
|
||||
def list_tables(self) -> list[dict]:
|
||||
"""Tabelle vettoriali disponibili: `{table_name, vector_dimensions, …}`."""
|
||||
return self._call("list_tables", {}) or []
|
||||
|
||||
def existing_hashes(self, table_name: str, kinds: list[str]) -> dict[str, str]:
|
||||
"""Hash correnti per sync incrementale su una tabella vector allowlisted.
|
||||
|
||||
RPC attesa: `existing_vector_hashes(table_name, kinds)` -> righe
|
||||
`{record_key, content_hash}`.
|
||||
"""
|
||||
rows = self._call(
|
||||
"existing_vector_hashes",
|
||||
{"table_name": table_name, "kinds": kinds},
|
||||
) or []
|
||||
return {row["record_key"]: row["content_hash"] for row in rows}
|
||||
|
||||
def upsert_records(self, table_name: str, rows: list[dict]) -> int:
|
||||
"""Upsert controllato di record vettoriali già embeddati.
|
||||
|
||||
RPC attesa: `upsert_vector_records(table_name, rows)` -> `{upserted: N}` o righe.
|
||||
Non espone delete/clear: il cleanup distruttivo resta solo-server.
|
||||
"""
|
||||
payload = self._call(
|
||||
"upsert_vector_records",
|
||||
{"table_name": table_name, "rows": rows},
|
||||
)
|
||||
if payload is None:
|
||||
return len(rows)
|
||||
if isinstance(payload, dict):
|
||||
return int(payload.get("upserted", len(rows)))
|
||||
return len(payload) if isinstance(payload, list) else len(rows)
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Scrittura controllata del pgvector via REST.
|
||||
|
||||
Usata dalle postazioni remote solo quando e' configurata una seconda API key di scrittura.
|
||||
Mantiene l'upsert incrementale del VectorStore diretto, ma non esegue delete/clear: le
|
||||
operazioni distruttive restano solo-server via connessione Postgres diretta.
|
||||
"""
|
||||
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
from tht.vectorstore.rest_client import VectorRestClient
|
||||
from tht.vectorstore.store import SyncStats, content_hash
|
||||
|
||||
|
||||
KIND_TO_TABLE = {
|
||||
"schema_table": "schema_records",
|
||||
"schema_column": "schema_records",
|
||||
"evidence": "evidence",
|
||||
"memory": "memory",
|
||||
}
|
||||
TABLE_TO_KINDS = {
|
||||
"schema_records": {"schema_table", "schema_column"},
|
||||
"evidence": {"evidence"},
|
||||
"memory": {"memory"},
|
||||
}
|
||||
|
||||
|
||||
def pack_metadata(record: VectorRecord) -> dict:
|
||||
"""Impacchetta nel metadata tutta la semantica letta poi da `search_similar`."""
|
||||
return {
|
||||
"kind": record.kind,
|
||||
"ref": record.ref,
|
||||
"record_key": record.id,
|
||||
"title": record.title,
|
||||
"content": record.content,
|
||||
**record.metadata,
|
||||
}
|
||||
|
||||
|
||||
class RestVectorWriter:
|
||||
"""Writer table-scoped via RPC REST allowlist.
|
||||
|
||||
Il metodo `sync` e' volutamente upsert-only: aggiorna/aggiunge record, conta gli stale,
|
||||
ma non li elimina. Per cleanup completo usare i comandi server-side con `vector_db`.
|
||||
"""
|
||||
|
||||
def __init__(self, client: VectorRestClient, table: str):
|
||||
if table not in TABLE_TO_KINDS:
|
||||
raise ValueError(f"Tabella vector non supportata per scrittura REST: {table}")
|
||||
self.client = client
|
||||
self.table = table
|
||||
|
||||
def existing_hashes(self, kinds: set[str]) -> dict[str, str]:
|
||||
allowed = TABLE_TO_KINDS[self.table]
|
||||
bad = kinds - allowed
|
||||
if bad:
|
||||
raise ValueError(
|
||||
f"Kind non ammessi per vectors.{self.table}: {', '.join(sorted(bad))}"
|
||||
)
|
||||
return self.client.existing_hashes(self.table, sorted(kinds))
|
||||
|
||||
def sync(self, records: list[VectorRecord], embedder, kinds: set[str]) -> SyncStats:
|
||||
stats = SyncStats()
|
||||
existing = self.existing_hashes(kinds)
|
||||
to_embed: list[VectorRecord] = []
|
||||
for record in records:
|
||||
h = content_hash(record.content)
|
||||
if record.id not in existing:
|
||||
to_embed.append(record)
|
||||
stats.added += 1
|
||||
elif existing[record.id] != h:
|
||||
to_embed.append(record)
|
||||
stats.updated += 1
|
||||
else:
|
||||
stats.unchanged += 1
|
||||
|
||||
stats.deleted = 0
|
||||
vectors = embedder.embed_documents([r.content for r in to_embed]) if to_embed else []
|
||||
rows = [
|
||||
{
|
||||
"record_key": record.id,
|
||||
"kind": record.kind,
|
||||
"content_hash": content_hash(record.content),
|
||||
"metadata": pack_metadata(record),
|
||||
"embedding": vector,
|
||||
}
|
||||
for record, vector in zip(to_embed, vectors)
|
||||
]
|
||||
if rows:
|
||||
self.client.upsert_records(self.table, rows)
|
||||
# Gli stale non vengono cancellati in REST writer: restano responsabilita' server-side.
|
||||
return stats
|
||||
@@ -0,0 +1,191 @@
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import Engine, text
|
||||
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
|
||||
|
||||
def content_hash(content: str) -> str:
|
||||
return hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
|
||||
def _to_vector_literal(vec: list[float]) -> str:
|
||||
return "[" + ",".join(f"{x:.8f}" for x in vec) + "]"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncStats:
|
||||
added: int = 0
|
||||
updated: int = 0
|
||||
deleted: int = 0
|
||||
unchanged: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class VectorHit:
|
||||
id: str
|
||||
kind: str
|
||||
ref: str
|
||||
title: str
|
||||
content: str
|
||||
metadata: dict
|
||||
similarity: float
|
||||
|
||||
|
||||
def hit_from_metadata(similarity: float, metadata: dict | None) -> VectorHit:
|
||||
"""Ricostruisce un VectorHit dal solo `metadata` (più la similarity). È l'unico modo
|
||||
disponibile leggendo via REST (`search_similar` ritorna id/similarity/metadata), e viene
|
||||
usato anche dalla lettura diretta per avere un'unica logica. Tollerante: usa default sui
|
||||
campi assenti (es. metadata estranei della tabella fake remota)."""
|
||||
md = metadata or {}
|
||||
return VectorHit(
|
||||
id=md.get("record_key", ""),
|
||||
kind=md.get("kind", ""),
|
||||
ref=md.get("ref", ""),
|
||||
title=md.get("title", ""),
|
||||
content=md.get("content", ""),
|
||||
metadata=md,
|
||||
similarity=float(similarity),
|
||||
)
|
||||
|
||||
|
||||
class VectorStore:
|
||||
"""Tabella pgvector table-scoped: scrittura diretta (loading) su una tabella dello schema
|
||||
`vectors`. La lettura via REST avviene su `search_similar`; questo store serve al loading e
|
||||
alla lettura diretta (dev/test). Il contratto della tabella remota richiede `id` (BIGSERIAL),
|
||||
`embedding vector(N)` e `metadata jsonb`; le colonne extra (`record_key`, `kind`,
|
||||
`content_hash`) servono solo al loader e non sono esposte dalla REST."""
|
||||
|
||||
def __init__(
|
||||
self, engine: Engine, schema: str = "vectors", table: str = "records", dim: int = 768
|
||||
):
|
||||
self.engine = engine
|
||||
self.schema = schema
|
||||
self.dim = dim
|
||||
self._table = f"{schema}.{table}"
|
||||
|
||||
def init_schema(self) -> None:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {self.schema}"))
|
||||
conn.execute(text(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self._table} (
|
||||
id bigserial PRIMARY KEY,
|
||||
record_key text UNIQUE NOT NULL,
|
||||
kind text NOT NULL,
|
||||
content_hash text NOT NULL,
|
||||
metadata jsonb NOT NULL DEFAULT '{{}}',
|
||||
embedding vector({self.dim}) NOT NULL,
|
||||
indexed_at timestamptz NOT NULL DEFAULT now()
|
||||
)
|
||||
"""))
|
||||
conn.execute(text(
|
||||
f"CREATE INDEX IF NOT EXISTS {self._idx('embedding')} ON {self._table} "
|
||||
f"USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 200)"
|
||||
))
|
||||
conn.execute(text(
|
||||
f"CREATE INDEX IF NOT EXISTS {self._idx('kind')} ON {self._table} (kind)"
|
||||
))
|
||||
# GRANT al ruolo di sola lettura della REST, solo se esiste (assente in test/locale).
|
||||
conn.execute(text(f"""
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vector_reader') THEN
|
||||
EXECUTE 'GRANT SELECT ON {self._table} TO vector_reader';
|
||||
END IF;
|
||||
END $$;
|
||||
"""))
|
||||
|
||||
def _idx(self, suffix: str) -> str:
|
||||
return f"{self._table.replace('.', '_')}_{suffix}_idx"
|
||||
|
||||
def clear(self) -> None:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(f"DELETE FROM {self._table}"))
|
||||
|
||||
def existing_hashes(self, kinds: set[str]) -> dict[str, str]:
|
||||
q = text(
|
||||
f"SELECT record_key, content_hash FROM {self._table} WHERE kind = ANY(:kinds)"
|
||||
)
|
||||
with self.engine.connect() as conn:
|
||||
return dict(conn.execute(q, {"kinds": list(kinds)}).fetchall())
|
||||
|
||||
def sync(self, records: list[VectorRecord], embedder, kinds: set[str]) -> SyncStats:
|
||||
"""Allinea l'indice ai record correnti (per i kind dati): embedda solo il nuovo
|
||||
o il modificato, elimina cio' che non esiste piu'."""
|
||||
stats = SyncStats()
|
||||
existing = self.existing_hashes(kinds)
|
||||
current_ids = {r.id for r in records}
|
||||
|
||||
to_embed: list[VectorRecord] = []
|
||||
for r in records:
|
||||
h = content_hash(r.content)
|
||||
if r.id not in existing:
|
||||
to_embed.append(r)
|
||||
stats.added += 1
|
||||
elif existing[r.id] != h:
|
||||
to_embed.append(r)
|
||||
stats.updated += 1
|
||||
else:
|
||||
stats.unchanged += 1
|
||||
|
||||
vectors = embedder.embed_documents([r.content for r in to_embed]) if to_embed else []
|
||||
|
||||
upsert = text(f"""
|
||||
INSERT INTO {self._table}
|
||||
(record_key, kind, content_hash, metadata, embedding)
|
||||
VALUES
|
||||
(:record_key, :kind, :content_hash, CAST(:metadata AS jsonb),
|
||||
CAST(:embedding AS vector))
|
||||
ON CONFLICT (record_key) DO UPDATE SET
|
||||
kind = EXCLUDED.kind, content_hash = EXCLUDED.content_hash,
|
||||
metadata = EXCLUDED.metadata, embedding = EXCLUDED.embedding,
|
||||
indexed_at = now()
|
||||
""")
|
||||
stale = [i for i in existing if i not in current_ids]
|
||||
with self.engine.begin() as conn:
|
||||
for r, vec in zip(to_embed, vectors):
|
||||
conn.execute(upsert, {
|
||||
"record_key": r.id, "kind": r.kind,
|
||||
"content_hash": content_hash(r.content),
|
||||
"metadata": json.dumps(_pack_metadata(r)),
|
||||
"embedding": _to_vector_literal(vec),
|
||||
})
|
||||
if stale:
|
||||
conn.execute(
|
||||
text(f"DELETE FROM {self._table} WHERE record_key = ANY(:ids)"),
|
||||
{"ids": stale},
|
||||
)
|
||||
stats.deleted = len(stale)
|
||||
return stats
|
||||
|
||||
def search(
|
||||
self, query_vec: list[float], top_n: int = 10, kinds: list[str] | None = None
|
||||
) -> list[VectorHit]:
|
||||
where = "WHERE kind = ANY(:kinds)" if kinds else ""
|
||||
q = text(f"""
|
||||
SELECT metadata, 1 - (embedding <=> CAST(:q AS vector)) AS similarity
|
||||
FROM {self._table}
|
||||
{where}
|
||||
ORDER BY embedding <=> CAST(:q AS vector)
|
||||
LIMIT :top_n
|
||||
""")
|
||||
params: dict = {"q": _to_vector_literal(query_vec), "top_n": top_n}
|
||||
if kinds:
|
||||
params["kinds"] = kinds
|
||||
with self.engine.connect() as conn:
|
||||
rows = conn.execute(q, params).fetchall()
|
||||
return [hit_from_metadata(r.similarity, r.metadata) for r in rows]
|
||||
|
||||
|
||||
def _pack_metadata(r: VectorRecord) -> dict:
|
||||
"""Impacchetta nel `metadata` (unica colonna letta via REST) tutta la semantica Thoth."""
|
||||
return {
|
||||
"kind": r.kind,
|
||||
"ref": r.ref,
|
||||
"record_key": r.id,
|
||||
"title": r.title,
|
||||
"content": r.content,
|
||||
**r.metadata,
|
||||
}
|
||||
Reference in New Issue
Block a user