feat: harden runtime readiness and session workflow
This commit is contained in:
@@ -27,7 +27,7 @@ def build_dwh(cfg: Config) -> DwhAdapter:
|
||||
|
||||
|
||||
def build_vector_store(cfg: Config, *, require_write: bool = False) -> VectorStore:
|
||||
"""Build the vector adapter, optionally requiring an HTTP writer credential."""
|
||||
"""Build the vector adapter, optionally requiring write capability."""
|
||||
resource = cfg.vectors
|
||||
if resource is None:
|
||||
raise ConfigError("Risorsa vectors non configurata")
|
||||
@@ -35,11 +35,17 @@ def build_vector_store(cfg: Config, *, require_write: bool = False) -> VectorSto
|
||||
match resource.type:
|
||||
case "pgvector_direct":
|
||||
reader = resource.reader or resource.connection
|
||||
if require_write and resource.writer is None:
|
||||
# Legacy server workspaces use one RW `vector_db` connection. Keep
|
||||
# that deployment contract without turning a workstation's legacy
|
||||
# compatibility connection into an implicit writer.
|
||||
writer = resource.writer or (
|
||||
resource.connection if cfg.profile == "server" else None
|
||||
)
|
||||
if require_write and writer is None:
|
||||
raise ConfigError("Vector writer non configurato per pgvector_direct")
|
||||
return PgVectorStore(
|
||||
reader,
|
||||
resource.writer,
|
||||
writer,
|
||||
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
|
||||
)
|
||||
case "thoth_vector_http":
|
||||
|
||||
@@ -48,6 +48,39 @@ def _cosine_operator(schema: str) -> sql.Composed:
|
||||
return sql.SQL("OPERATOR({}.<=>)").format(sql.Identifier(schema))
|
||||
|
||||
|
||||
def _vector_sql_names(cursor, table_schema: str, collection: str) -> tuple[str, str]:
|
||||
"""Discover pgvector type and operator namespaces from the embedding column."""
|
||||
cursor.execute(
|
||||
"""SELECT type_ns.nspname, operator_ns.nspname
|
||||
FROM pg_catalog.pg_attribute attribute
|
||||
JOIN pg_catalog.pg_class table_class
|
||||
ON table_class.oid = attribute.attrelid
|
||||
JOIN pg_catalog.pg_namespace table_ns
|
||||
ON table_ns.oid = table_class.relnamespace
|
||||
JOIN pg_catalog.pg_type vector_type
|
||||
ON vector_type.oid = attribute.atttypid
|
||||
JOIN pg_catalog.pg_namespace type_ns
|
||||
ON type_ns.oid = vector_type.typnamespace
|
||||
JOIN pg_catalog.pg_operator cosine
|
||||
ON cosine.oprname = %s
|
||||
AND cosine.oprleft = vector_type.oid
|
||||
AND cosine.oprright = vector_type.oid
|
||||
JOIN pg_catalog.pg_namespace operator_ns
|
||||
ON operator_ns.oid = cosine.oprnamespace
|
||||
WHERE table_ns.nspname = %s
|
||||
AND table_class.relname = %s
|
||||
AND attribute.attname = %s
|
||||
AND NOT attribute.attisdropped
|
||||
ORDER BY cosine.oid
|
||||
LIMIT 1""",
|
||||
("<=>", table_schema, collection, "embedding"),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
raise VectorStoreError(f"Collection {collection} has no usable pgvector embedding")
|
||||
return row[0], row[1]
|
||||
|
||||
|
||||
def _validate_collection_kinds(collection: str, kinds: list[str]) -> None:
|
||||
invalid = set(kinds) - COLLECTION_KINDS[collection]
|
||||
if invalid:
|
||||
@@ -261,6 +294,9 @@ class PgVectorStore:
|
||||
with raw.cursor() as cursor:
|
||||
for collection in collections:
|
||||
table = _collection(self._schema, collection)
|
||||
type_schema, operator_schema = _vector_sql_names(
|
||||
cursor, self._schema, collection
|
||||
)
|
||||
collection_kinds = (
|
||||
sorted(set(kinds) & COLLECTION_KINDS[collection]) if kinds else None
|
||||
)
|
||||
@@ -293,12 +329,12 @@ class PgVectorStore:
|
||||
"SELECT metadata, 1 - (embedding {} %s::{}) AS similarity "
|
||||
"FROM {}{} ORDER BY embedding {} %s::{}, record_key LIMIT %s"
|
||||
).format(
|
||||
_cosine_operator(self._schema),
|
||||
_vector_type(self._schema),
|
||||
_cosine_operator(operator_schema),
|
||||
_vector_type(type_schema),
|
||||
table,
|
||||
where,
|
||||
_cosine_operator(self._schema),
|
||||
_vector_type(self._schema),
|
||||
_cosine_operator(operator_schema),
|
||||
_vector_type(type_schema),
|
||||
)
|
||||
params = [_vector_literal(embedding)]
|
||||
params.extend(filter_params)
|
||||
@@ -352,19 +388,20 @@ class PgVectorStore:
|
||||
and len(write_record.embedding) != self._expected_dimension
|
||||
):
|
||||
raise VectorStoreError("Embedding dimension does not match configured dimension")
|
||||
insert = sql.SQL(
|
||||
"INSERT INTO {} (record_key, kind, content_hash, metadata, embedding) "
|
||||
"VALUES (%s, %s, %s, %s::jsonb, %s::{}) "
|
||||
"ON CONFLICT (record_key) DO NOTHING"
|
||||
).format(table, _vector_type(self._schema))
|
||||
update = sql.SQL(
|
||||
"UPDATE {} SET kind = %s, content_hash = %s, metadata = %s::jsonb, "
|
||||
"embedding = %s::{}, indexed_at = pg_catalog.now() WHERE record_key = %s"
|
||||
).format(table, _vector_type(self._schema))
|
||||
raw = None
|
||||
try:
|
||||
raw = engine.raw_connection()
|
||||
with raw.cursor() as cursor:
|
||||
type_schema, _ = _vector_sql_names(cursor, self._schema, collection)
|
||||
insert = sql.SQL(
|
||||
"INSERT INTO {} (record_key, kind, content_hash, metadata, embedding) "
|
||||
"VALUES (%s, %s, %s, %s::jsonb, %s::{}) "
|
||||
"ON CONFLICT (record_key) DO NOTHING"
|
||||
).format(table, _vector_type(type_schema))
|
||||
update = sql.SQL(
|
||||
"UPDATE {} SET kind = %s, content_hash = %s, metadata = %s::jsonb, "
|
||||
"embedding = %s::{}, indexed_at = pg_catalog.now() WHERE record_key = %s"
|
||||
).format(table, _vector_type(type_schema))
|
||||
for write_record in records:
|
||||
record = write_record.record
|
||||
metadata = {
|
||||
|
||||
@@ -145,11 +145,11 @@ def save_one_cmd(
|
||||
json_out: bool = typer.Option(False, "--json", help="Output JSON (per Pi)."),
|
||||
config: Path = CONFIG_OPT,
|
||||
) -> None:
|
||||
"""Upsert mirato (una riga) della memoria di una decisione su pgvector via writer key (D11).
|
||||
"""Upsert mirato (una riga) della memoria di una decisione su pgvector (D11).
|
||||
|
||||
Promuove la decisione nel registro locale (idempotente) e fa un singolo upsert
|
||||
remoto con dedup hash client-side -- niente full-resync. Abilita il salvataggio
|
||||
di una memoria da postazione remota (workstation) con la sola writer key.
|
||||
con dedup hash client-side -- niente full-resync. Il factory seleziona il writer
|
||||
REST su workstation oppure il writer pgvector diretto sul profilo server.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
@@ -160,13 +160,6 @@ def save_one_cmd(
|
||||
cfg = _load_config_or_exit(config)
|
||||
manifest = load_session_or_exit(cfg, session)
|
||||
require_vector_write_allowed(cfg, "memory save-one")
|
||||
if not has_vector_write_rest(cfg):
|
||||
typer.secho(
|
||||
"ERRORE: `memory save-one` richiede la sezione `vector_write_rest` con una "
|
||||
"API key di upsert nel workspace yaml (upsert remoto via writer key).",
|
||||
fg=typer.colors.RED, err=True,
|
||||
)
|
||||
raise typer.Exit(code=4)
|
||||
store = build_vector_store(cfg, require_write=True)
|
||||
|
||||
sdir = session_dir(cfg, session)
|
||||
@@ -516,6 +509,7 @@ def solved_search_cmd(
|
||||
from rich.table import Table
|
||||
|
||||
from tht.cli.vector_cmd import make_embedder, open_searcher
|
||||
from tht.ports.vector import VectorReadUnavailable
|
||||
from tht.solved import SOLVED_KIND
|
||||
from tht.vectorstore.embeddings import EmbeddingsError
|
||||
from tht.vectorstore.rest_client import VectorRestError
|
||||
@@ -529,7 +523,7 @@ def solved_search_cmd(
|
||||
searcher = open_searcher(cfg)
|
||||
embedder = make_embedder(cfg.embeddings)
|
||||
hits = searcher.search(embedder.embed_query(question), top_n=top, kinds=[SOLVED_KIND])
|
||||
except (VectorRestError, EmbeddingsError, OperationalError) as e:
|
||||
except (VectorRestError, VectorReadUnavailable, EmbeddingsError, OperationalError) as e:
|
||||
typer.secho(
|
||||
f"ATTENZIONE: exemplar non disponibili ({e}). Prosegui senza.",
|
||||
fg=typer.colors.YELLOW, err=True,
|
||||
|
||||
@@ -10,6 +10,7 @@ from pathlib import Path
|
||||
import typer
|
||||
|
||||
from tht.cli.config_cmd import CONFIG_OPT
|
||||
from tht.config import workspace_id_from_path
|
||||
|
||||
|
||||
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
|
||||
@@ -86,7 +87,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
||||
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
return pipeline.run_as_job(
|
||||
workspace_id=config.stem.lower().replace(".", "-").replace("_", "-"),
|
||||
workspace_id=workspace_id_from_path(config),
|
||||
workspace_root=corpus_root.parent,
|
||||
config_fingerprint=fingerprint(cfg.model_dump_json()),
|
||||
input_fingerprint=fingerprint(config.resolve().as_posix()),
|
||||
@@ -115,7 +116,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
|
||||
pipeline_version="evidence-v1",
|
||||
retain_published_generations=cfg.vector.retain_published_generations,
|
||||
)
|
||||
pipeline.workspace_id = config.stem.lower().replace(".", "-").replace("_", "-")
|
||||
pipeline.workspace_id = workspace_id_from_path(config)
|
||||
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
|
||||
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import typer
|
||||
|
||||
from tht.cli.config_cmd import CONFIG_OPT
|
||||
from tht.cli.schema_cmd import _load_config_or_exit
|
||||
from tht.config import workspace_id_from_path
|
||||
|
||||
KIND_MAP = {
|
||||
"evidence": ["evidence"],
|
||||
@@ -59,7 +60,7 @@ def search_cmd(
|
||||
cfg = _load_config_or_exit(config)
|
||||
from tht.search.evidence import validate_corpus_workspace
|
||||
|
||||
workspace_id = config.stem.lower().replace(".", "-").replace("_", "-")
|
||||
workspace_id = workspace_id_from_path(config)
|
||||
validate_corpus_workspace(cfg, workspace_id)
|
||||
dwh_snapshot = _leased_dwh_snapshot(cfg, ctx)
|
||||
require_vector_cfg(cfg)
|
||||
@@ -254,6 +255,7 @@ def pack_cmd(
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from tht.cli.vector_cmd import make_embedder, open_searcher, require_vector_cfg
|
||||
from tht.ports.vector import VectorReadUnavailable
|
||||
from tht.search import combined_search, schema_tables
|
||||
from tht.solved import SOLVED_KIND
|
||||
from tht.vectorstore.embeddings import EmbeddingsError
|
||||
@@ -262,7 +264,7 @@ def pack_cmd(
|
||||
cfg = _load_config_or_exit(config)
|
||||
from tht.search.evidence import validate_corpus_workspace
|
||||
|
||||
workspace_id = config.stem.lower().replace(".", "-").replace("_", "-")
|
||||
workspace_id = workspace_id_from_path(config)
|
||||
validate_corpus_workspace(cfg, workspace_id)
|
||||
dwh_snapshot = _leased_dwh_snapshot(cfg, ctx)
|
||||
require_vector_cfg(cfg)
|
||||
@@ -271,7 +273,7 @@ def pack_cmd(
|
||||
evidence: list[dict] = []
|
||||
solved: list[dict] = []
|
||||
warnings: list[str] = []
|
||||
degrade = (VectorRestError, EmbeddingsError, OperationalError)
|
||||
degrade = (VectorRestError, VectorReadUnavailable, EmbeddingsError, OperationalError)
|
||||
|
||||
vec = None
|
||||
searcher = embedder = None
|
||||
|
||||
@@ -201,6 +201,27 @@ def show_cmd(
|
||||
typer.echo(f"schema_linking.json: {'presente' if linking.exists() else 'assente'}")
|
||||
|
||||
|
||||
@session_app.command("retrieval-pack")
|
||||
def retrieval_pack_cmd(
|
||||
session_id: str = typer.Argument(...),
|
||||
config: Path = CONFIG_OPT,
|
||||
) -> None:
|
||||
"""Emette su stdout il retrieval pack persistito della sessione."""
|
||||
cfg = _load_config_or_exit(config)
|
||||
load_session_or_exit(cfg, session_id)
|
||||
path = session_dir(cfg, session_id) / "retrieval_pack.md"
|
||||
try:
|
||||
content = path.read_text()
|
||||
except OSError as exc:
|
||||
typer.secho(
|
||||
f"ERRORE: retrieval pack non disponibile per la sessione {session_id}: {exc}",
|
||||
fg=typer.colors.RED,
|
||||
err=True,
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
typer.echo(content, nl=False)
|
||||
|
||||
|
||||
@session_app.command("close")
|
||||
def close_cmd(session_id: str = typer.Argument(...), config: Path = CONFIG_OPT) -> None:
|
||||
"""Chiude la sessione (status=closed)."""
|
||||
@@ -212,6 +233,27 @@ def close_cmd(session_id: str = typer.Argument(...), config: Path = CONFIG_OPT)
|
||||
typer.secho(f"OK: sessione {session_id} chiusa.", fg=typer.colors.GREEN)
|
||||
|
||||
|
||||
@session_app.command("fail")
|
||||
def fail_cmd(session_id: str = typer.Argument(...), config: Path = CONFIG_OPT) -> None:
|
||||
"""Registra un arresto del sistema non recuperabile automaticamente."""
|
||||
from tht.session.store import fail_session
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
load_session_or_exit(cfg, session_id)
|
||||
fail_session(session_id, cfg.paths.sessions)
|
||||
typer.secho(f"OK: sessione {session_id} marcata failed.", fg=typer.colors.RED)
|
||||
|
||||
@session_app.command("reopen")
|
||||
def reopen_cmd(session_id: str = typer.Argument(...), config: Path = CONFIG_OPT) -> None:
|
||||
"""Riapre una sessione per una ripresa manuale."""
|
||||
from tht.session.store import reopen_session
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
load_session_or_exit(cfg, session_id)
|
||||
reopen_session(session_id, cfg.paths.sessions)
|
||||
typer.secho(f"OK: sessione {session_id} riaperta.", fg=typer.colors.GREEN)
|
||||
|
||||
|
||||
@session_app.command("set-name")
|
||||
def set_name_cmd(
|
||||
session_id: str = typer.Argument(...),
|
||||
|
||||
@@ -327,6 +327,11 @@ class Config(BaseModel):
|
||||
return translated
|
||||
|
||||
|
||||
def workspace_id_from_path(path: Path) -> str:
|
||||
"""Return the stable workspace identity, resolving deployment aliases first."""
|
||||
return path.resolve().stem.lower().replace(".", "-").replace("_", "-")
|
||||
|
||||
|
||||
def load_config(path: Path) -> Config:
|
||||
if not path.exists():
|
||||
raise ConfigError(f"File di configurazione non trovato: {path}")
|
||||
@@ -378,7 +383,7 @@ def load_config(path: Path) -> Config:
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
cfg._workspace_id = path.stem.lower().replace(".", "-").replace("_", "-")
|
||||
cfg._workspace_id = workspace_id_from_path(path)
|
||||
cfg._config_source = path.resolve().as_posix()
|
||||
return cfg
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ class _YamlModel(BaseModel):
|
||||
class SessionManifest(_YamlModel):
|
||||
id: str
|
||||
created_at: datetime
|
||||
status: Literal["open", "closed", "finalized"] = "open"
|
||||
status: Literal["open", "closed", "failed", "finalized"] = "open"
|
||||
question: str
|
||||
database: str
|
||||
db_schema: str = Field(alias="schema")
|
||||
|
||||
@@ -242,6 +242,25 @@ def close_session(session_id: str, sessions_root: Path) -> SessionManifest:
|
||||
return manifest
|
||||
|
||||
|
||||
def fail_session(session_id: str, sessions_root: Path) -> SessionManifest:
|
||||
"""Record a fatal managed-runtime failure without losing phase artifacts."""
|
||||
manifest = load_session(session_id, sessions_root)
|
||||
manifest.status = "failed"
|
||||
manifest.updated_at = datetime.now(UTC)
|
||||
manifest.updated_by = current_author()
|
||||
manifest.to_yaml(sessions_root / session_id / MANIFEST)
|
||||
return manifest
|
||||
|
||||
def reopen_session(session_id: str, sessions_root: Path) -> SessionManifest:
|
||||
"""Mark a manually resumed session active again."""
|
||||
manifest = load_session(session_id, sessions_root)
|
||||
manifest.status = "open"
|
||||
manifest.updated_at = datetime.now(UTC)
|
||||
manifest.updated_by = current_author()
|
||||
manifest.to_yaml(sessions_root / session_id / MANIFEST)
|
||||
return manifest
|
||||
|
||||
|
||||
def _save_touched(manifest: SessionManifest, sessions_root: Path) -> SessionManifest:
|
||||
"""Persist `manifest` updating updated_at/updated_by (single save path for mutations)."""
|
||||
manifest.updated_at = datetime.now(UTC)
|
||||
|
||||
Reference in New Issue
Block a user