feat(harness): port backend Onda 0 — vendor, lshindex, sqlcheck, execute, rest/exec, ctetest, report, datamart
8 moduli leaf portati verbatim da ChironeWp3 con rename psdwp3→tht: - vendor/thoth_lsh (MinHash/LSH, leaf puro datasketch+tqdm) + VENDORED.md - lshindex/ (build/save/load/query, dipende vendor + LshConfig) - sqlcheck/ (validate_sql, leaf ExecutionConfig+mschema) - execute/ + execute/warnings (run_controlled/explain, leaf sqlglot+sqlalchemy) - rest/execute + rest/explain (REST variants, dipendono execute+rest.client) - ctetest (CTE test records, leaf sqlglot+pydantic) - report (validation report rendering, dipende execute+sqlcheck) - datamart (stub NotImplementedError) Verifica: import smoke catena completa OK, pytest 109 passed. Deps (datasketch, sqlglot, sqlalchemy, pydantic, requests, tqdm) già in pyproject. VENDORED.md neutralizzato (riferimenti PsdWp3→Thoth).
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
import sqlglot
|
||||
from sqlalchemy import Engine, text
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
|
||||
|
||||
class ExecutionError(Exception):
|
||||
"""Errore di esecuzione con messaggio leggibile per il reviewer."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecResult:
|
||||
columns: list[str]
|
||||
rows: list[tuple]
|
||||
execution_ms: int
|
||||
truncated: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlanSummary:
|
||||
total_cost: float
|
||||
plan_rows: int
|
||||
node_types: list[str]
|
||||
|
||||
|
||||
def _inject_limit(sql: str, limit: int) -> tuple[str, bool]:
|
||||
"""Aggiunge LIMIT limit+1 se assente (il +1 serve a rilevare il troncamento).
|
||||
Se la query ha gia' un suo LIMIT, lo si rispetta."""
|
||||
ast = sqlglot.parse_one(sql, read="postgres")
|
||||
# solo le query (SELECT/UNION & co.) accettano un LIMIT: per qualunque altro
|
||||
# statement non si inietta nulla e la transazione READ ONLY lo rifiutera' (rete 2).
|
||||
if not isinstance(ast, sqlglot.exp.Query) or ast.args.get("limit") is not None:
|
||||
return sql, False
|
||||
return ast.limit(limit + 1).sql(dialect="postgres"), True
|
||||
|
||||
|
||||
def _translate_error(e: DBAPIError) -> ExecutionError:
|
||||
msg = str(e.orig)
|
||||
if "statement timeout" in msg or "canceling statement" in msg:
|
||||
return ExecutionError("timeout: la query ha superato statement_timeout_ms")
|
||||
if "read-only" in msg:
|
||||
return ExecutionError("rifiutato: statement non eseguibile in transazione read-only")
|
||||
return ExecutionError(f"errore SQL: {msg}")
|
||||
|
||||
|
||||
def run_controlled(engine: Engine, sql: str, *, limit: int, timeout_ms: int) -> ExecResult:
|
||||
"""Esecuzione nelle 4 reti: utente RO (a monte), transazione READ ONLY,
|
||||
statement_timeout, LIMIT iniettato via AST."""
|
||||
final_sql, injected = _inject_limit(sql, limit)
|
||||
start = time.monotonic()
|
||||
with engine.connect() as conn:
|
||||
trans = conn.begin()
|
||||
try:
|
||||
conn.execute(text("SET TRANSACTION READ ONLY"))
|
||||
conn.execute(text(f"SET LOCAL statement_timeout = {int(timeout_ms)}"))
|
||||
result = conn.execute(text(final_sql))
|
||||
columns = list(result.keys())
|
||||
rows = [tuple(r) for r in result.fetchall()]
|
||||
except DBAPIError as e:
|
||||
raise _translate_error(e) from e
|
||||
finally:
|
||||
trans.rollback()
|
||||
elapsed_ms = int((time.monotonic() - start) * 1000)
|
||||
truncated = injected and len(rows) > limit
|
||||
return ExecResult(
|
||||
columns=columns, rows=rows[:limit], execution_ms=elapsed_ms, truncated=truncated
|
||||
)
|
||||
|
||||
|
||||
def explain(engine: Engine, sql: str, *, timeout_ms: int) -> PlanSummary:
|
||||
"""EXPLAIN (FORMAT JSON), mai ANALYZE: il piano si stima, non si esegue."""
|
||||
with engine.connect() as conn:
|
||||
trans = conn.begin()
|
||||
try:
|
||||
conn.execute(text("SET TRANSACTION READ ONLY"))
|
||||
conn.execute(text(f"SET LOCAL statement_timeout = {int(timeout_ms)}"))
|
||||
row = conn.execute(text(f"EXPLAIN (FORMAT JSON) {sql}")).fetchone()
|
||||
except DBAPIError as e:
|
||||
raise _translate_error(e) from e
|
||||
finally:
|
||||
trans.rollback()
|
||||
payload = row[0]
|
||||
if isinstance(payload, str):
|
||||
payload = json.loads(payload)
|
||||
plan = payload[0]["Plan"]
|
||||
|
||||
node_types: list[str] = []
|
||||
|
||||
def collect(node: dict) -> None:
|
||||
node_types.append(node.get("Node Type", "?"))
|
||||
for child in node.get("Plans", []):
|
||||
collect(child)
|
||||
|
||||
collect(plan)
|
||||
return PlanSummary(
|
||||
total_cost=float(plan.get("Total Cost", 0.0)),
|
||||
plan_rows=int(plan.get("Plan Rows", 0)),
|
||||
node_types=node_types,
|
||||
)
|
||||
@@ -0,0 +1,40 @@
|
||||
from sqlglot import exp
|
||||
|
||||
from tht.config import ExecutionConfig
|
||||
from tht.execute import ExecResult, PlanSummary
|
||||
|
||||
|
||||
def runtime_warnings(result: ExecResult, cfg: ExecutionConfig) -> list[str]:
|
||||
warnings: list[str] = []
|
||||
if not result.rows:
|
||||
warnings.append("risultato vuoto: verificare filtri e join")
|
||||
if result.truncated:
|
||||
warnings.append("righe = limit: risultato troncato dalla preview")
|
||||
if result.execution_ms > cfg.warn_execution_ms:
|
||||
warnings.append(
|
||||
f"tempo di esecuzione elevato: {result.execution_ms} ms "
|
||||
f"(soglia {cfg.warn_execution_ms} ms)"
|
||||
)
|
||||
return warnings
|
||||
|
||||
|
||||
def plan_warnings(plan: PlanSummary, cfg: ExecutionConfig) -> list[str]:
|
||||
warnings: list[str] = []
|
||||
if plan.plan_rows > cfg.warn_plan_rows:
|
||||
warnings.append(
|
||||
f"righe stimate dal piano: {plan.plan_rows} (soglia {cfg.warn_plan_rows}) — "
|
||||
f"possibile join molto ampio o cardinalita' anomala"
|
||||
)
|
||||
return warnings
|
||||
|
||||
|
||||
def static_warnings(ast: exp.Expression) -> list[str]:
|
||||
"""Euristica cast sospetti: CAST su colonne dentro WHERE (non-sargable)."""
|
||||
warnings: list[str] = []
|
||||
for where in ast.find_all(exp.Where):
|
||||
for cast in where.find_all(exp.Cast):
|
||||
if cast.find(exp.Column) is not None:
|
||||
warnings.append(
|
||||
f"cast su colonna nel WHERE (possibile non-sargable): {cast.sql()}"
|
||||
)
|
||||
return warnings
|
||||
Reference in New Issue
Block a user