mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(core): preserve adapter command contracts

Browse Source
This commit is contained in:
marcopan
2026-07-11 21:00:50 +02:00
parent dbbab6d005
commit a35efa16de
20 changed files with 163 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/adapters/dwh/postgres.py
+14 -7
View File
@@ -4,7 +4,7 @@ from tht.config import DatabaseConfig
from sqlalchemy.exc import SQLAlchemyError
from tht.db import execute, sampling
from tht.db.connection import make_engine, ping
from tht.db.connection import can_create_in_schema, make_engine, ping, writable_tables
from tht.db.introspect import introspect
from tht.execute import ExecResult, PlanSummary
from tht.mschema.models import PhysicalSchema
@@ -14,32 +14,39 @@ from tht.ports.dwh import DistinctValues, DwhCapabilities, DwhHealth
class PostgresDwhAdapter:
capabilities = DwhCapabilities()
def __init__(self, config: DatabaseConfig):
def __init__(self, config: DatabaseConfig, *, statement_timeout_ms: int = 30_000):
self._config = config
self._engine = make_engine(config)
self._statement_timeout_ms = statement_timeout_ms
def health(self) -> DwhHealth:
try:
ping(self._engine)
except SQLAlchemyError as exc:
return DwhHealth(ok=False, detail=str(exc))
return DwhHealth(ok=True)
writable = tuple(writable_tables(self._engine, self._config.db_schema))
can_create = can_create_in_schema(self._engine, self._config.db_schema)
return DwhHealth(ok=True, database=self._config.database, schema=self._config.db_schema,
read_only=not writable and not can_create,
writable_tables=writable, can_create=can_create)
def introspect(self) -> PhysicalSchema:
return introspect(self._engine, self._config.database, self._config.db_schema)
def run_query(self, sql: str, *, limit: int) -> ExecResult:
return execute.run_query(self._engine, sql, limit=limit)
return execute.run_query(
self._engine, sql, limit=limit, timeout_ms=self._statement_timeout_ms
)
def explain(self, sql: str) -> PlanSummary:
return execute.explain(self._engine, sql)
return execute.explain(self._engine, sql, timeout_ms=self._statement_timeout_ms)
def sample_column(self, table: str, column: str, *, limit: int) -> list[object]:
return sampling.sample_column(
self._engine, self._config.db_schema, table, column, limit=limit
)
def distinct_values(self, table: str, column: str) -> DistinctValues:
def distinct_values(self, table: str, column: str, *, limit: int) -> DistinctValues:
return sampling.distinct_values(
self._engine, self._config.db_schema, table, column
self._engine, self._config.db_schema, table, column, max_values=limit
)
harness/tht/adapters/dwh/thoth_rest.py
+6 -4
View File
@@ -22,8 +22,10 @@ class ThothRestDwhAdapter:
result = self._client.ping()
except RestError as exc:
return DwhHealth(ok=False, detail=str(exc))
ok = bool(result.get("db_connected", True))
return DwhHealth(ok=ok, detail=None if ok else str(result))
ok = bool(result.get("db_connected") and result.get("schema_accessible"))
return DwhHealth(ok=ok, detail=None if ok else str(result),
database=self._database.database, schema=self._database.db_schema,
endpoint=self._client.cfg.base_url, read_only=True)
def introspect(self) -> PhysicalSchema:
return introspect_rest(
@@ -44,10 +46,10 @@ class ThothRestDwhAdapter:
except RestError as exc:
raise ExecutionError(str(exc)) from exc
def distinct_values(self, table: str, column: str) -> DistinctValues:
def distinct_values(self, table: str, column: str, *, limit: int) -> DistinctValues:
try:
return sampling.distinct_values_rest(
self._client, self._database.db_schema, table, column
self._client, self._database.db_schema, table, column, max_values=limit
)
except RestError as exc:
raise ExecutionError(str(exc)) from exc
harness/tht/adapters/factory.py
+4 -1
View File
@@ -14,7 +14,10 @@ def build_dwh(cfg: Config) -> DwhAdapter:
resource = cfg.dwh
match resource.type:
case "postgres_direct":
return PostgresDwhAdapter(resource.connection)
return PostgresDwhAdapter(
resource.connection,
statement_timeout_ms=cfg.execution.statement_timeout_ms,
)
case "thoth_rest":
return ThothRestDwhAdapter(resource.database, resource.endpoint)
case other: # pragma: no cover - Pydantic's discriminator rejects this first.
harness/tht/cli/db_cmd.py
+16 -48
View File
@@ -1,43 +1,14 @@
from pathlib import Path
import typer
from sqlalchemy.exc import OperationalError
from tht.adapters.dwh import ThothRestDwhAdapter
from tht.adapters.factory import build_dwh
from tht.cli.config_cmd import CONFIG_OPT
from tht.config import ConfigError, load_config
from tht.db.connection import can_create_in_schema, make_engine, ping, writable_tables
from tht.db.fetch_ca import CaFetchError, describe_pem, fetch_chain_pem, parse_host_port
db_app = typer.Typer(help="Operazioni sul database target")
def _ping_rest(adapter: ThothRestDwhAdapter, cfg, schema: str) -> None:
"""Health check via REST. Il read-only è garantito strutturalmente dall'API
(ammette solo SELECT/WITH): non serve il controllo dei privilegi di scrittura."""
from tht.rest.client import RestError
try:
info = adapter._client.ping()
except RestError as e:
typer.secho(f"ERRORE di connessione: {e}", fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)
if not info.get("db_connected") or not info.get("schema_accessible"):
typer.secho(
f"ERRORE: DWH non accessibile via REST (risposta: {info}).",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(code=1)
typer.secho(
f"OK: connesso via REST a {cfg.rest.base_url} (schema {schema})", fg=typer.colors.GREEN
)
typer.secho(
"OK: accesso read-only garantito dall'API (solo SELECT/WITH).", fg=typer.colors.GREEN
)
@db_app.command("ping")
def ping_cmd(config: Path = CONFIG_OPT) -> None:
"""Testa la connessione e verifica che l'utente sia effettivamente read-only."""
@@ -46,29 +17,26 @@ def ping_cmd(config: Path = CONFIG_OPT) -> None:
except ConfigError as e:
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)
schema = cfg.database.db_schema
adapter = build_dwh(cfg)
if isinstance(adapter, ThothRestDwhAdapter):
_ping_rest(adapter, cfg, schema)
return
engine = make_engine(cfg.database)
try:
ping(engine)
except OperationalError as e:
typer.secho(f"ERRORE di connessione: {e.orig}", fg=typer.colors.RED, err=True)
health = build_dwh(cfg).health()
if not health.ok:
typer.secho(f"ERRORE di connessione: {health.detail}", fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)
typer.secho(f"OK: connesso a {cfg.database.database} (schema {schema})", fg=typer.colors.GREEN)
writable = writable_tables(engine, schema)
can_create = can_create_in_schema(engine, schema)
if writable or can_create:
if health.endpoint:
typer.secho(f"OK: connesso via REST a {health.endpoint} (schema {health.schema})",
fg=typer.colors.GREEN)
typer.secho("OK: accesso read-only garantito dall'API (solo SELECT/WITH).",
fg=typer.colors.GREEN)
return
typer.secho(f"OK: connesso a {health.database} (schema {health.schema})",
fg=typer.colors.GREEN)
if not health.read_only:
typer.secho(
f"ERRORE: l'utente '{cfg.database.user}' NON e' read-only.", fg=typer.colors.RED, err=True
)
if writable:
typer.echo(f" Tabelle scrivibili: {', '.join(writable[:10])}", err=True)
if can_create:
typer.echo(f" L'utente puo' creare oggetti nello schema {schema}.", err=True)
if health.writable_tables:
typer.echo(f" Tabelle scrivibili: {', '.join(health.writable_tables[:10])}", err=True)
if health.can_create:
typer.echo(f" L'utente puo' creare oggetti nello schema {health.schema}.", err=True)
typer.echo(" Crea un ruolo read-only con scripts/create_readonly_role.sql.", err=True)
raise typer.Exit(code=2)
typer.secho("OK: l'utente e' read-only sullo schema target.", fg=typer.colors.GREEN)
harness/tht/cli/lsh_cmd.py
+3 -1
View File
@@ -47,7 +47,9 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
if not is_text_type(column.type) or not effective_eligibility(column, ann_col)[0]:
continue
try:
distinct = dwh.distinct_values(table_name, column_name)
distinct = dwh.distinct_values(
table_name, column_name, limit=cfg.lsh.max_values_per_column
)
except Exception as e:
skipped.append(SkippedColumn(table_name, column_name, f"errore: {e}"))
continue
harness/tht/cli/memory_cmd.py
+4 -7
View File
@@ -156,7 +156,6 @@ def save_one_cmd(
from tht.adapters.factory import build_vector_store
from tht.cli.vector_cmd import make_embedder
from tht.memory import load_registry, promote, save_one_memory
from tht.vectorstore.rest_client import VectorRestClient
cfg = _load_config_or_exit(config)
manifest = load_session_or_exit(cfg, session)
@@ -168,7 +167,7 @@ def save_one_cmd(
fg=typer.colors.RED, err=True,
)
raise typer.Exit(code=4)
build_vector_store(cfg, require_write=True)
store = build_vector_store(cfg, require_write=True)
sdir = session_dir(cfg, session)
# Promuove la decisione scelta nel registro locale (idempotente: salta se gia' presente
@@ -176,9 +175,8 @@ def save_one_cmd(
promote(sdir, manifest, seqs=[decision], registry_path=registry_path(cfg))
records = [r for r in load_registry(registry_path(cfg)) if r.session_id == manifest.id]
writer = VectorRestClient(cfg.vector_write_rest)
embedder = make_embedder(cfg.embeddings)
count = save_one_memory(records, decision, writer=writer, embedder=embedder)
count = save_one_memory(records, decision, store=store, embedder=embedder)
msg = (
f"{count} memoria salvata su pgvector (decision_seq {decision})."
@@ -455,21 +453,20 @@ def index_solved_session(cfg, session_id: str) -> int:
from tht.cli.sql_cmd import promoted_tables_for
from tht.cli.vector_cmd import make_embedder
from tht.solved import build_solved_record, save_solved_question
from tht.vectorstore.rest_client import VectorRestClient
if not has_vector_write_rest(cfg):
raise RuntimeError(
"vector_write_rest assente: la coppia domanda->SQL si indicizza solo con la "
"writer key configurata nel workspace yaml"
)
build_vector_store(cfg, require_write=True)
store = build_vector_store(cfg, require_write=True)
manifest = load_session_or_exit(cfg, session_id)
record = build_solved_record(
session_dir(cfg, session_id), manifest, promoted_tables_for(cfg, session_id)
)
return save_solved_question(
record,
writer=VectorRestClient(cfg.vector_write_rest),
store=store,
embedder=make_embedder(cfg.embeddings),
)
harness/tht/cli/schema_cmd.py
+19 -12
View File
@@ -1,4 +1,5 @@
from pathlib import Path
import logging
import typer
from tht.adapters.factory import build_dwh
@@ -8,6 +9,23 @@ from tht.db.sampling import is_text_type
from tht.mschema.eligibility import classify_all
schema_app = typer.Typer(help="Gestione mschema (rappresentazione canonica dello schema)")
logger = logging.getLogger(__name__)
def _add_examples(dwh, phys, examples) -> None:
for table_name, table in phys.tables.items():
for column_name, column in table.columns.items():
if not is_text_type(column.type):
continue
try:
sampled = dwh.sample_column(
table_name, column_name, limit=examples.max_per_column
)
except Exception as exc:
logger.warning("Campionamento saltato per %s.%s: %s",
table_name, column_name, exc)
continue
column.examples = [str(value) for value in sampled if value not in (None, "")]
def _load_config_or_exit(config: Path):
@@ -64,18 +82,7 @@ def introspect_cmd(
try:
dwh = build_dwh(cfg)
phys = dwh.introspect()
for table_name, table in phys.tables.items():
for column_name, column in table.columns.items():
if is_text_type(column.type):
column.examples = [
str(value)
for value in dwh.sample_column(
table_name,
column_name,
limit=cfg.examples.max_per_column,
)
if value not in (None, "")
]
_add_examples(dwh, phys, cfg.examples)
classify_all(phys, cfg.eligibility)
except Exception as e:
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
harness/tht/db/execute.py
+4 -4
View File
@@ -13,15 +13,15 @@ from tht.execute import (
DEFAULT_TIMEOUT_MS = 30_000
def run_query(engine: Engine, sql: str, *, limit: int) -> ExecResult:
def run_query(engine: Engine, sql: str, *, limit: int, timeout_ms: int = DEFAULT_TIMEOUT_MS) -> ExecResult:
limit = require_positive_int(limit, name="limit")
return run_controlled(
engine,
sql,
limit=limit,
timeout_ms=DEFAULT_TIMEOUT_MS,
timeout_ms=timeout_ms,
)
def explain(engine: Engine, sql: str) -> PlanSummary:
return _explain(engine, sql, timeout_ms=DEFAULT_TIMEOUT_MS)
def explain(engine: Engine, sql: str, *, timeout_ms: int = DEFAULT_TIMEOUT_MS) -> PlanSummary:
return _explain(engine, sql, timeout_ms=timeout_ms)
harness/tht/memory.py
+8 -13
View File
@@ -255,7 +255,7 @@ def memory_vector_record_for_decision(
def save_one_memory(
records: list[MemoryRecord], decision_seq: int, *, writer, embedder
records: list[MemoryRecord], decision_seq: int, *, store, embedder
) -> int:
"""Targeted one-row upsert of a promoted decision to pgvector via the writer key
(spec D11). This is NOT a full vectorstore resync: it embeds and pushes a single
@@ -267,27 +267,22 @@ def save_one_memory(
the writer's existing_vector_hashes; the embedding (Ollama round-trip) and the
upsert are skipped when the content is unchanged. Idempotent by construction.
`writer` is a VectorRestClient (writer key); `embedder` an embeddings client.
`store` is the configured writable VectorStore; `embedder` an embeddings client.
The destructive cleanup (sync's delete-stale step) is intentionally absent: it
remains a server-side-only operation via the direct vectordb connection.
"""
from tht.vectorstore.rest_writer import pack_metadata
from tht.ports.vector import VectorWriteRecord
from tht.vectorstore.store import content_hash
record = memory_vector_record_for_decision(records, decision_seq)
if record is None:
return 0
new_hash = content_hash(record.content)
existing = writer.existing_hashes("memory", ["memory"])
existing = store.existing_hashes("memory", ["memory"])
if existing.get(record.id) == new_hash:
return 0 # unchanged: skip embedding + upsert
embedding = embedder.embed_documents([record.content])[0]
row = {
"record_key": record.id,
"kind": record.kind,
"content_hash": new_hash,
"metadata": pack_metadata(record),
"embedding": embedding,
}
return writer.upsert_records("memory", [row])
return store.upsert(
"memory",
[VectorWriteRecord(record=record, embedding=embedding, content_hash=new_hash)],
)
harness/tht/ports/dwh.py
+7 -1
View File
@@ -19,6 +19,12 @@ class DwhCapabilities:
class DwhHealth:
ok: bool
detail: str | None = None
database: str | None = None
schema: str | None = None
endpoint: str | None = None
read_only: bool | None = None
writable_tables: tuple[str, ...] = ()
can_create: bool = False
@dataclass(frozen=True)
@@ -46,4 +52,4 @@ class DwhAdapter(Protocol):
def sample_column(self, table: str, column: str, *, limit: int) -> list[object]: ...
def distinct_values(self, table: str, column: str) -> DistinctValues: ...
def distinct_values(self, table: str, column: str, *, limit: int) -> DistinctValues: ...
harness/tht/solved.py
+7 -10
View File
@@ -43,25 +43,22 @@ def _solved_hash(record: VectorRecord) -> str:
return content_hash(record.content + "\n" + str(record.metadata.get("sql", "")))
def save_solved_question(record: VectorRecord, *, writer, embedder) -> int:
def save_solved_question(record: VectorRecord, *, store, embedder) -> int:
"""Upsert one-row della coppia domanda->SQL via writer key (stesso pattern di
save_one_memory, spec D11): hash dedup client-side, embedding solo se domanda
o SQL sono cambiati. `writer` e' un VectorRestClient (writer key). Ritorna il
numero di righe upsertate (0 = invariata)."""
from tht.vectorstore.rest_writer import pack_metadata
from tht.ports.vector import VectorWriteRecord
new_hash = _solved_hash(record)
existing = writer.existing_hashes("memory", [SOLVED_KIND])
existing = store.existing_hashes("memory", [SOLVED_KIND])
if existing.get(record.id) == new_hash:
return 0
embedding = embedder.embed_documents([record.content])[0]
return writer.upsert_records("memory", [{
"record_key": record.id,
"kind": record.kind,
"content_hash": new_hash,
"metadata": pack_metadata(record),
"embedding": embedding,
}])
return store.upsert(
"memory",
[VectorWriteRecord(record=record, embedding=embedding, content_hash=new_hash)],
)
class SolvedIndexError(Exception):