feat(vector): add direct pgvector adapter

This commit is contained in:
2026-07-12 01:01:15 +02:00
parent ebdd3aa2c5
commit b09341f07e
7 changed files with 515 additions and 35 deletions
+30 -9
View File
@@ -18,6 +18,7 @@ class ConfigError(Exception):
def _expand_env(value: Any) -> Any:
if isinstance(value, str):
def repl(m: re.Match) -> str:
var = m.group(1)
if var not in os.environ:
@@ -85,7 +86,16 @@ DwhResourceConfig = Annotated[
class PgvectorDirectConfig(BaseModel):
type: Literal["pgvector_direct"]
connection: DatabaseConfig
reader: DatabaseConfig | None = None
writer: DatabaseConfig | None = None
# Deprecated compatibility: a single direct connection historically meant read-only.
connection: DatabaseConfig | None = None
@model_validator(mode="after")
def validate_connections(self):
if self.reader is None and self.writer is None and self.connection is None:
raise ValueError("pgvector_direct requires a reader or writer connection")
return self
class ThothVectorHttpConfig(BaseModel):
@@ -134,9 +144,9 @@ class LshConfig(BaseModel):
class EligibilityConfig(BaseModel):
# Soglie del principio di column eligibility (testo ampio ignorato ovunque).
max_declared_len: int = 128 # char/varchar dichiarati <= soglia: eligible senza campionare
max_avg_length: int = 40 # fallback data-driven: lunghezza media valori campionati
max_sampled_len: int = 200 # fallback data-driven: lunghezza massima valore campionato
max_declared_len: int = 128 # char/varchar dichiarati <= soglia: eligible senza campionare
max_avg_length: int = 40 # fallback data-driven: lunghezza media valori campionati
max_sampled_len: int = 200 # fallback data-driven: lunghezza massima valore campionato
# Colonne di servizio sempre ignorate per nome (match case-insensitive), a prescindere
# dal tipo: metadati ETL/audit non analitici (es. timestamp di ultimo aggiornamento).
ignore_columns: list[str] = ["etl_last_update"]
@@ -168,7 +178,7 @@ class VectorConfig(BaseModel):
class SearchConfig(BaseModel):
rrf_k: int = 60
top_schema_tables: int = 12 # default `--top` per `tht search --kind schema` (n. tabelle)
top_schema_tables: int = 12 # default `--top` per `tht search --kind schema` (n. tabelle)
schema_chunk_pool: int = 150 # chunk tabella/colonna fusi prima dell'aggregazione a tabella
@@ -181,9 +191,17 @@ class ExecutionConfig(BaseModel):
max_aggregate_cells: int = 20
max_export_rows: int = 100000
forbidden_functions: list[str] = [
"setval", "nextval", "pg_advisory_lock", "pg_advisory_xact_lock",
"dblink", "dblink_exec", "pg_terminate_backend", "pg_cancel_backend",
"lo_import", "lo_export", "pg_reload_conf",
"setval",
"nextval",
"pg_advisory_lock",
"pg_advisory_xact_lock",
"dblink",
"dblink_exec",
"pg_terminate_backend",
"pg_cancel_backend",
"lo_import",
"lo_export",
"pg_reload_conf",
]
@@ -302,7 +320,10 @@ def _populate_legacy_views(raw: dict[str, Any]) -> None:
vectors = raw.get("vectors")
if isinstance(vectors, dict):
if vectors.get("type") == "pgvector_direct":
raw.setdefault("vector_db", vectors["connection"])
raw.setdefault(
"vector_db",
vectors.get("writer") or vectors.get("reader") or vectors.get("connection"),
)
elif vectors.get("type") == "thoth_vector_http":
raw.setdefault("vector_rest", vectors.get("reader"))
raw.setdefault("vector_write_rest", vectors.get("writer"))